From 8d843133ded877264410be2783fb1e49b55524b1 Mon Sep 17 00:00:00 2001 From: AccentuSoft Date: Thu, 17 Feb 2022 02:49:55 +0200 Subject: [PATCH] Improve SNScrape twitter user info extraction. Rework financial entities. Remove redundant resolution 'name' fields from resolutions. --- Core/Entities/Financials.xml | 7 ++- Modules/CryptoCurrency/BlockChainAPI.py | 12 ++--- Modules/EDGAR_&_EFD/CompanyInfo.py | 18 ++----- Modules/EDGAR_&_EFD/Edgar.xml | 14 +---- Modules/EDGAR_&_EFD/FramesLookUp.py | 11 ++-- Modules/EDGAR_&_EFD/Get10KForms.py | 2 - Modules/EDGAR_&_EFD/Get10QForms.py | 2 - Modules/EDGAR_&_EFD/Get13FForms.py | 1 - Modules/EDGAR_&_EFD/Get20FForms.py | 2 - Modules/EDGAR_&_EFD/Get40FForms.py | 2 - Modules/EDGAR_&_EFD/Get6KForms.py | 2 - Modules/EDGAR_&_EFD/Get8KForms.py | 2 - Modules/EDGAR_&_EFD/GetDForms.py | 1 - Modules/EDGAR_&_EFD/GetN8FForms.py | 1 - Modules/EDGAR_&_EFD/GetNMFP2Forms.py | 6 +-- Modules/GitHubRSAFinder/DorkingMethod.py | 1 - .../OrgSearch_GitAllSecrets.py | 8 --- Modules/Pulsedive_API/IndicatorsInfo.py | 46 ++++++++-------- Modules/Pulsedive_API/ThreatInfo.py | 10 ++-- Modules/SNScrape/TwitterUser.py | 52 ++++++++++++++++++- 20 files changed, 95 insertions(+), 105 deletions(-) diff --git a/Core/Entities/Financials.xml b/Core/Entities/Financials.xml index 7a05e44..8771e79 100644 --- a/Core/Entities/Financials.xml +++ b/Core/Entities/Financials.xml @@ -18,15 +18,14 @@ Currency.svg - + - Ticker - Amount + Ticker ID Share.svg - + Label diff --git a/Modules/CryptoCurrency/BlockChainAPI.py b/Modules/CryptoCurrency/BlockChainAPI.py index d75afa9..5e2c988 100644 --- a/Modules/CryptoCurrency/BlockChainAPI.py +++ b/Modules/CryptoCurrency/BlockChainAPI.py @@ -49,30 +49,26 @@ class BlockChainAPI: 'Currency Type': 'BTC', 'Notes': 'sent by address', 'Entity Type': 'Currency'}, - {uid: {'Resolution': 'sent by address', - 'Name': 'sent by address', + {uid: {'Resolution': 'Sent by address', 'Notes': ''}}]) returnResults.append( [{'Amount': str(int(getReceivedByAddress.text) / 100000000), 'Currency Type': 'BTC', 'Notes': 'received by address', 'Entity Type': 'Currency'}, - {uid: {'Resolution': 'received by address', - 'Name': 'received by address', + {uid: {'Resolution': 'Received by address', 'Notes': ''}}]) returnResults.append( [{'Amount': str(int(addressBalance.text) / 100000000), 'Currency Type': 'BTC', 'Notes': 'address balance', 'Entity Type': 'Currency'}, - {uid: {'Resolution': 'address balance', - 'Name': 'address balance', + {uid: {'Resolution': 'Address balance', 'Notes': ''}}]) returnResults.append( [{'Date': str(dateCreated), 'Notes': 'first seen date', 'Entity Type': 'Date'}, - {uid: {'Resolution': 'first seen date', - 'Name': 'first seen date', + {uid: {'Resolution': 'First seen date', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/CompanyInfo.py b/Modules/EDGAR_&_EFD/CompanyInfo.py index 77b4d32..39b2a8c 100644 --- a/Modules/EDGAR_&_EFD/CompanyInfo.py +++ b/Modules/EDGAR_&_EFD/CompanyInfo.py @@ -41,43 +41,37 @@ class CompanyInfo: exchanges = data['exchanges'] for exchange in exchanges: returnResults.append([{'Exchange Name': exchange, - 'Entity Type': 'Exchanges'}, + 'Entity Type': 'Exchange'}, {uid: {'Resolution': 'Exchange', - 'Name': 'Exchange', 'Notes': ''}}]) tickers = data['tickers'] for ticker in tickers: - returnResults.append([{'Ticker Name': ticker, - 'Entity Type': 'Tickers'}, - {uid: {'Resolution': 'Exchange', - 'Name': 'Exchange', + returnResults.append([{'Ticker ID': ticker, + 'Entity Type': 'Ticker'}, + {uid: {'Resolution': 'Ticker', 'Notes': ''}}]) if data['insiderTransactionForOwnerExists'] == 1: returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists', 'Entity Type': 'Phrase'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) else: returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists', 'Entity Type': 'Phrase'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if data['insiderTransactionForIssuerExists'] == 1: returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists', 'Entity Type': 'Phrase'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) else: returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists', 'Entity Type': 'Phrase'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if data['sic'] is not None: @@ -85,13 +79,11 @@ class CompanyInfo: 'Description': data['sicDescription'], 'Entity Type': 'SIC'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if data['ein'] is not None: returnResults.append([{'EIN': str(data['ein']), 'Entity Type': 'EIN'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if data['addresses'] is not None: returnResults.append([{'Street Address': data['addresses']['mailing']['street1'], @@ -100,7 +92,6 @@ class CompanyInfo: 'Locality': data['addresses']['mailing']['city'], 'Entity Type': 'Address'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']: @@ -110,6 +101,5 @@ class CompanyInfo: 'Locality': data['addresses']['business']['city'], 'Entity Type': 'Address'}, {uid: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Edgar.xml b/Modules/EDGAR_&_EFD/Edgar.xml index 26d91d7..86892c9 100644 --- a/Modules/EDGAR_&_EFD/Edgar.xml +++ b/Modules/EDGAR_&_EFD/Edgar.xml @@ -66,16 +66,11 @@ Gross Proceeds Used - - - Ticker Name - - - + Exchange Name - + SIC @@ -102,11 +97,6 @@ ISINID - - - Amount - - Field Name diff --git a/Modules/EDGAR_&_EFD/FramesLookUp.py b/Modules/EDGAR_&_EFD/FramesLookUp.py index 1134f7f..cdb8f04 100644 --- a/Modules/EDGAR_&_EFD/FramesLookUp.py +++ b/Modules/EDGAR_&_EFD/FramesLookUp.py @@ -9,7 +9,7 @@ class FramesLookUp: originTypes = {'Form Field'} - resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Shares'} + resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Phrase'} parameters = { 'Quarter': {'description': 'Please Ensure that the selected Taxonomy matches the Form Field you typed', @@ -66,31 +66,26 @@ class FramesLookUp: returnResults.append([{'Company Name': data['data'][i]['entityName'], 'Entity Type': 'Company'}, {uid: {'Resolution': 'Edgar Company', - 'Name': 'Edgar Company', 'Notes': ''}}]) returnResults.append([{'CIK': str(data['data'][i]['cik']).zfill(10), 'Entity Type': 'Edgar ID'}, {index_of_child: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) returnResults.append([{'Country Name': data['data'][i]['loc'], 'Entity Type': 'Country'}, {index_of_child: {'Resolution': '', - 'Name': '', 'Notes': ''}}]) if unit == 'USD': returnResults.append([{'Amount': str(data['data'][i]['val']), 'Currency Type': 'USD', 'Entity Type': 'Currency'}, {index_of_child: {'Resolution': 'Form Filed Value', - 'Name': 'Edgar ID', 'Notes': ''}}]) elif unit == 'shares': - returnResults.append([{'Amount': str(data['data'][i]['val']), - 'Entity Type': 'Shares'}, + returnResults.append([{'Phrase': 'Number of Shares: ' + str(data['data'][i]['val']), + 'Entity Type': 'Phrase'}, {index_of_child: {'Resolution': 'Form Filed Value', - 'Name': 'Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get10KForms.py b/Modules/EDGAR_&_EFD/Get10KForms.py index 074b707..61fa420 100644 --- a/Modules/EDGAR_&_EFD/Get10KForms.py +++ b/Modules/EDGAR_&_EFD/Get10KForms.py @@ -70,7 +70,6 @@ class Get10KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '10-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -90,7 +89,6 @@ class Get10KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '10-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get10QForms.py b/Modules/EDGAR_&_EFD/Get10QForms.py index d2faf6a..0c9dd1a 100644 --- a/Modules/EDGAR_&_EFD/Get10QForms.py +++ b/Modules/EDGAR_&_EFD/Get10QForms.py @@ -69,7 +69,6 @@ class Get10QForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '10-Q Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -88,7 +87,6 @@ class Get10QForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '10-Q Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get13FForms.py b/Modules/EDGAR_&_EFD/Get13FForms.py index 7057069..2a6aaff 100644 --- a/Modules/EDGAR_&_EFD/Get13FForms.py +++ b/Modules/EDGAR_&_EFD/Get13FForms.py @@ -100,6 +100,5 @@ class Get13FForms: 'Entity Type': 'Form13F'}, {uid: {'Resolution': 'Form13F', - 'Name': 'Form13F', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get20FForms.py b/Modules/EDGAR_&_EFD/Get20FForms.py index a7e005d..9f6cb32 100644 --- a/Modules/EDGAR_&_EFD/Get20FForms.py +++ b/Modules/EDGAR_&_EFD/Get20FForms.py @@ -69,7 +69,6 @@ class Get20FForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '20-F Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -89,7 +88,6 @@ class Get20FForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '20-F Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get40FForms.py b/Modules/EDGAR_&_EFD/Get40FForms.py index b7a7793..aeb7aaf 100644 --- a/Modules/EDGAR_&_EFD/Get40FForms.py +++ b/Modules/EDGAR_&_EFD/Get40FForms.py @@ -70,7 +70,6 @@ class Get40FForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '40-F Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -90,7 +89,6 @@ class Get40FForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '40-F Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get6KForms.py b/Modules/EDGAR_&_EFD/Get6KForms.py index ba2b378..69414fa 100644 --- a/Modules/EDGAR_&_EFD/Get6KForms.py +++ b/Modules/EDGAR_&_EFD/Get6KForms.py @@ -72,7 +72,6 @@ class Get6KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '6-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -92,7 +91,6 @@ class Get6KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '6-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/Get8KForms.py b/Modules/EDGAR_&_EFD/Get8KForms.py index a0769e6..5359b11 100644 --- a/Modules/EDGAR_&_EFD/Get8KForms.py +++ b/Modules/EDGAR_&_EFD/Get8KForms.py @@ -72,7 +72,6 @@ class Get8KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '8-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) if list(data['facts'][form][i]['units'].keys())[0] == 'shares': @@ -92,7 +91,6 @@ class Get8KForms: 'Entity Type': 'Form Field'}, {uid: {'Resolution': '8-K Field', - 'Name': 'CIK Edgar ID', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/GetDForms.py b/Modules/EDGAR_&_EFD/GetDForms.py index 934ffdf..6cc1f01 100644 --- a/Modules/EDGAR_&_EFD/GetDForms.py +++ b/Modules/EDGAR_&_EFD/GetDForms.py @@ -114,7 +114,6 @@ class GetDForms: 'Entity Type': 'FormD'}, {uid: {'Resolution': 'D Form', - 'Name': 'D Form', 'Notes': ''}}]) people = data['edgarSubmission']['relatedPersonsList']['relatedPersonInfo'] diff --git a/Modules/EDGAR_&_EFD/GetN8FForms.py b/Modules/EDGAR_&_EFD/GetN8FForms.py index 9282f51..e84f0f1 100644 --- a/Modules/EDGAR_&_EFD/GetN8FForms.py +++ b/Modules/EDGAR_&_EFD/GetN8FForms.py @@ -70,6 +70,5 @@ class GetN8FForms: returnResults.append([{'URL': anchor, 'Entity Type': 'Website'}, {uid: {'Resolution': 'N-8F Form', - 'Name': 'N-8F Form', 'Notes': ''}}]) return returnResults diff --git a/Modules/EDGAR_&_EFD/GetNMFP2Forms.py b/Modules/EDGAR_&_EFD/GetNMFP2Forms.py index 8bcdbe1..b4ae688 100644 --- a/Modules/EDGAR_&_EFD/GetNMFP2Forms.py +++ b/Modules/EDGAR_&_EFD/GetNMFP2Forms.py @@ -148,7 +148,6 @@ class GetNMFP2Forms: 'Friday 5': 'NO Value in Daily Measure', 'Entity Type': 'FormNMFP2'}, {uid: {'Resolution': field, - 'Name': 'FormNMFP2', 'Notes': ''}}]) else: returnResults.append([{'Field Name': f'N-MFP2:({field})' + ' ' @@ -165,7 +164,6 @@ class GetNMFP2Forms: 'ns3:fridayWeek5'], 'Entity Type': 'FormNMFP2'}, {uid: {'Resolution': field, - 'Name': 'FormNMFP2', 'Notes': ''}}]) classLevelInfo = data['edgarSubmission']['formData']['classLevelInfo'] @@ -189,7 +187,6 @@ class GetNMFP2Forms: 'ns3:fridayWeek5'], 'Entity Type': 'FormNMFP2'}, {index_of_child: {'Resolution': 'Net Asset Per Share', - 'Name': 'FormNMFP2', 'Notes': ''}}]) for weekCount in range(1, 6): returnResults.append( @@ -262,8 +259,7 @@ class GetNMFP2Forms: 'Ctgry Investments Rprsnts Collateral': issuer['ctgryInvestmentsRprsntsCollateral'], 'Entity Type': 'Collateral Issuer'}, - {index_of_child: {'Resolution': 'Net Asset Per Share', - 'Name': 'Collateral Issuer', + {index_of_child: {'Resolution': 'Collateral Issuer', 'Notes': ''}}]) except KeyError: diff --git a/Modules/GitHubRSAFinder/DorkingMethod.py b/Modules/GitHubRSAFinder/DorkingMethod.py index 4ee4ee5..0a93c20 100644 --- a/Modules/GitHubRSAFinder/DorkingMethod.py +++ b/Modules/GitHubRSAFinder/DorkingMethod.py @@ -61,7 +61,6 @@ class DorkingMethod: [{'URL': anchor, 'Entity Type': 'Website'}, {uid: {'Resolution': 'RSA Key', - 'Name': 'RSA Key', 'Notes': ''}}]) return returnResults diff --git a/Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py b/Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py index 7999f7d..6a55361 100644 --- a/Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py +++ b/Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py @@ -71,14 +71,12 @@ class OrgSearch_GitAllSecrets: returnResults.append([{'Organisation Name': 'Org or User: ' + userOrg[1], 'Entity Type': 'GitHub Organisation'}, {uid: {'Resolution': 'Tool: repo-supervisor', - 'Name': 'Tool: repo-supervisor', 'Notes': ''}}]) child_of_child = len(returnResults) returnResults.append([{'Repository Name': userOrg[3], 'Entity Type': 'GitHub Repository'}, {index_of_child: {'Resolution': 'Repository of Organisation', - 'Name': 'Repository of Organisation', 'Notes': ''}}]) index += 1 @@ -94,7 +92,6 @@ class OrgSearch_GitAllSecrets: 'Notes': ansi_escape.sub('', hogSecret[hogIndex]), 'Entity Type': 'Phrase'}, {index_of_child: {'Resolution': 'GitHub Secret', - 'Name': 'GitHub Secret', 'Notes': ''}}]) # print(ansi_escape.sub('', hogSecret[hogIndex])) hogIndex += 1 @@ -102,19 +99,16 @@ class OrgSearch_GitAllSecrets: returnResults.append([{'Hash Value': ansi_escape.sub('', line), 'Entity Type': 'Hash'}, {childOfChild: {'Resolution': 'Tool: truffleHog', - 'Name': 'GitHub Hash', 'Notes': ''}}]) elif 'Filepath' in line: returnResults.append([{'Filepath': ansi_escape.sub('', line), 'Entity Type': 'GitHub FilePath'}, {childOfChild: {'Resolution': 'GitHub FilePath', - 'Name': 'GitHub FilePath', 'Notes': ''}}]) elif 'Branch' in line: returnResults.append([{'Branch': ansi_escape.sub('', line), 'Entity Type': 'GitHub Branch'}, {childOfChild: {'Resolution': 'GitHub Branch', - 'Name': 'GitHub Branch', 'Notes': ''}}]) for result in data: @@ -125,14 +119,12 @@ class OrgSearch_GitAllSecrets: returnResults.append([{'Filepath': secret.get('filepath'), 'Entity Type': 'GitHub FilePath'}, {child_of_child: {'Resolution': 'GitHub FilePath', - 'Name': 'GitHub FilePath', 'Notes': ''}}]) for scrt in secrets: returnResults.append([{'Secret': scrt, 'Entity Type': 'GitHub Secret'}, {child_child: {'Resolution': 'GitHub Secret', - 'Name': 'GitHub Secret', 'Notes': ''}}]) return returnResults diff --git a/Modules/Pulsedive_API/IndicatorsInfo.py b/Modules/Pulsedive_API/IndicatorsInfo.py index 886edb9..3dc5e12 100644 --- a/Modules/Pulsedive_API/IndicatorsInfo.py +++ b/Modules/Pulsedive_API/IndicatorsInfo.py @@ -61,23 +61,23 @@ class IndicatorsInfo: returnResults.append([{'Phrase': indicator['description'], 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Pulsedive Indicator Scan', - 'Name': 'Pulsedive Indicator Scan', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phrase': 'risk:' + indicator['risk'], 'Entity Type': 'Phrase'}, {index_of_child: {'Resolution': 'Risk', - 'Name': 'Risk', 'Notes': ''}}]) + 'Notes': ''}}]) for port in data["attributes"]["port"]: returnResults.append([{'Port': search_term + ':' + str(port), 'Entity Type': 'Port'}, {uid: {'Resolution': 'Ports Open', - 'Name': 'Ports Open', 'Notes': ''}}]) + 'Notes': ''}}]) for technology in data["attributes"]["technology"]: returnResults.append([{'Phrase': technology, 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Technologies Used', - 'Name': 'Technologies Used', 'Notes': ''}}]) + 'Notes': ''}}]) index_of_child = len(returnResults) @@ -87,17 +87,17 @@ class IndicatorsInfo: 'Postal Code': data["properties"]["geo"]['zip'], 'Entity Type': 'Address'}, {uid: {'Resolution': 'Address', - 'Name': 'Address', 'Notes': ''}}]) + 'Notes': ''}}]) elif data["properties"]["geo"].get('country') is not None: returnResults.append([{'Country Name': data["properties"]["geo"]['country'], 'Entity Type': 'Country'}, {index_of_child: {'Resolution': 'Country', - 'Name': 'Country', 'Notes': ''}}]) + 'Notes': ''}}]) elif data["properties"]["geo"].get('city') is not None: returnResults.append([{'City Name': data["properties"]["geo"]['city'], 'Entity Type': 'City'}, {index_of_child: {'Resolution': 'City', - 'Name': 'City', 'Notes': ''}}]) + 'Notes': ''}}]) dns_list = data["properties"]["dns"] for key in dns_list.keys(): @@ -112,12 +112,12 @@ class IndicatorsInfo: returnResults.append([{'Domain Name': value, 'Entity Type': 'Domain'}, {uid: {'Resolution': 'MX Record', - 'Name': 'MX Record', 'Notes': ''}}]) + 'Notes': ''}}]) elif dns_type[i] == "a": returnResults.append([{'IP Address': data["properties"]["dns"]["a"], 'Entity Type': 'IP Address'}, {index_of_child: {'Resolution': 'A Record', - 'Name': 'A Record', 'Notes': ''}}]) + 'Notes': ''}}]) elif dns_type[i] == "txt": for value in data["properties"]["dns"]['txt']: @@ -125,34 +125,34 @@ class IndicatorsInfo: 'Entity Type': 'Phrase', 'Notes': value}, {uid: {'Resolution': 'TXT Record', - 'Name': 'TXT Record', 'Notes': ''}}]) + 'Notes': ''}}]) elif dns_type[i] == "soa": returnResults.append([{'Domain Name': data["properties"]["dns"]['rname'], 'Entity Type': 'Domain'}, {uid: {'Resolution': 'Rname Record', - 'Name': 'Rname Record', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Domain Name': data["properties"]["dns"]['soa'], 'Entity Type': 'Domain'}, {index_of_child: {'Resolution': 'SOA Record', - 'Name': 'SOA Record', 'Notes': ''}}]) + 'Notes': ''}}]) elif dns_type[i] == "ns": for value in data["properties"]["dns"]['ns']: returnResults.append([{'Domain Name': value, 'Entity Type': 'Domain'}, {uid: {'Resolution': 'NS Record', - 'Name': 'NS Record', 'Notes': ''}}]) + 'Notes': ''}}]) if data["properties"].get("ssl") is not None: returnResults.append([{'Date': data["properties"]["ssl"]['expires'], 'Notes': 'expiration date of ssl certificate', 'Entity Type': 'Date'}, {uid: {'Resolution': 'expiration date of ssl certificate', - 'Name': 'expiration date of ssl certificate', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Company Name': data["properties"]["ssl"]['org'], 'Notes': 'issuer of certificate', 'Entity Type': 'Company'}, {uid: {'Resolution': 'issuer of certificate', - 'Name': 'issuer of certificate', 'Notes': ''}}]) + 'Notes': ''}}]) for key in data["properties"]["whois"].keys(): whois_type.append(key) @@ -164,27 +164,27 @@ class IndicatorsInfo: returnResults.append([{'Company Name': field[kind], 'Entity Type': 'Company'}, {uid: {'Resolution': 'Registrar Company', - 'Name': 'Registrar Company', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phrase': 'registry domain id' + ":" + field["registry domain id"], 'Entity Type': 'Phrase'}, {index_of_child: {'Resolution': 'Registry Domain ID', - 'Name': 'registry domain id', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phrase': "registrar iana id" + ":" + field["registrar iana id"], 'Entity Type': 'Phrase'}, {index_of_child: {'Resolution': 'Registrar IANA ID', - 'Name': 'registrar iana id', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Email Address': field["registrar abuse contact email"], 'Entity Type': 'Email Address'}, {index_of_child: {'Resolution': 'Registrar abuse contact email', - 'Name': 'Registrar abuse contact email', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phone Number': field["registrar abuse contact phone"], 'Entity Type': 'Phone Number'}, {index_of_child: {'Resolution': 'Registrar abuse contact phone', - 'Name': 'Registrar abuse contact phone', 'Notes': ''}}]) + 'Notes': ''}}]) elif kind == 'admin organization': index_of_child = len(returnResults) @@ -192,16 +192,16 @@ class IndicatorsInfo: 'Notes': 'admin organization', 'Entity Type': 'Company'}, {uid: {'Resolution': 'Admin Organisation', - 'Name': 'Admin Organisation', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Email Address': field['admin email'], 'Notes': 'admin email', 'Entity Type': 'Email Address'}, {index_of_child: {'Resolution': 'Admin Email Address', - 'Name': 'Admin Email Address', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phone Number': field['admin phone'], 'Notes': 'admin phone', 'Entity Type': 'Phone Number'}, {index_of_child: {'Resolution': 'Admin Phone Number', - 'Name': 'Admin Phone Number', 'Notes': ''}}]) + 'Notes': ''}}]) return returnResults diff --git a/Modules/Pulsedive_API/ThreatInfo.py b/Modules/Pulsedive_API/ThreatInfo.py index 45da726..615b9fd 100644 --- a/Modules/Pulsedive_API/ThreatInfo.py +++ b/Modules/Pulsedive_API/ThreatInfo.py @@ -60,11 +60,11 @@ class ThreatInfo: returnResults.append([{'Phrase': 'category: ' + data["category"], 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Category', - 'Name': 'Category', 'Notes': ''}}]) + 'Notes': ''}}]) returnResults.append([{'Phrase': 'risk: ' + data["risk"], 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Risk', - 'Name': 'Risk', 'Notes': ''}}]) + 'Notes': ''}}]) if linkNumbers == 0: linkNumbers = int(len(data["news"])) @@ -76,7 +76,7 @@ class ThreatInfo: 'Date Created': data["news"][i]['stamp'].replace(' ', 'T'), 'Entity Type': 'Website'}, {uid: {'Resolution': 'Related Article', - 'Name': 'Related Article', 'Notes': ''}}]) + 'Notes': ''}}]) if not data["summary"]["attributes"]: continue @@ -85,12 +85,12 @@ class ThreatInfo: returnResults.append([{'Phrase': key, 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Technologies Used', - 'Name': 'Technologies Used', 'Notes': ''}}]) + 'Notes': ''}}]) for key in data["summary"]["attributes"]["protocol"].keys(): returnResults.append([{'Phrase': key, 'Entity Type': 'Phrase'}, {uid: {'Resolution': 'Protocols Used', - 'Name': 'Protocols Used', 'Notes': ''}}]) + 'Notes': ''}}]) return returnResults diff --git a/Modules/SNScrape/TwitterUser.py b/Modules/SNScrape/TwitterUser.py index 31db1dd..631ef4c 100644 --- a/Modules/SNScrape/TwitterUser.py +++ b/Modules/SNScrape/TwitterUser.py @@ -144,7 +144,7 @@ class TwitterUser: 'User URL': item.user.url, 'Verified': str(item.user.verified), 'Display Name': item.user.displayname, - 'Location': item.user.location, + 'Location': str(item.user.location), 'Description': item.user.description, 'Protected': str(item.user.protected), 'Followers': str(item.user.followersCount), @@ -157,6 +157,7 @@ class TwitterUser: 'Icon': iconByteArrayFin, # If None, it will have the pic for Twitter User 'Date Created': item.user.created.isoformat()}, {uid: {'Resolution': 'Twitter User'}}]) + tweetIndex = len(returnResults) returnResults.append([{'Tweet ID': str(item.id), 'Tweet URL': item.url, 'Replies': str(item.replyCount), @@ -167,8 +168,55 @@ class TwitterUser: 'Place': str(item.place), 'Entity Type': 'Tweet', 'Date Created': item.date.isoformat(), - 'Notes': item.content}, + 'Notes': str(item.content)}, {childIndex: {'Resolution': 'Tweet'}}]) + if item.outlinks: + for link in item.outlinks: + returnResults.append([{'URL': link, + 'Entity Type': 'Website'}, + {tweetIndex: {'Resolution': 'External Link in Tweet'}}]) + if item.cashtags: + for cashtag in item.cashtags: + returnResults.append([{'Ticker ID': cashtag, + 'Entity Type': 'Ticker'}, + {tweetIndex: {'Resolution': 'Cashtag in Tweet'}}]) + if item.hashtags: + for hashtag in item.hashtags: + returnResults.append([{'Phrase': hashtag, + 'Entity Type': 'Phrase'}, + {tweetIndex: {'Resolution': 'Hashtag in Tweet'}}]) + if item.media: + for mediaItem in item.media: + if mediaItem.variants: + maxBitrate = 0 + bestVariant = None + for variant in mediaItem.variants: + if variant.bitrate and variant.bitrate > maxBitrate: + maxBitrate = variant.bitrate + bestVariant = variant + if bestVariant: + returnResults.append([{'URL': str(bestVariant.url), + 'Entity Type': 'Website'}, + {tweetIndex: {'Resolution': 'Video in Tweet'}}]) + else: + returnResults.append([{'URL': str(mediaItem.fullUrl), + 'Entity Type': 'Website'}, + {tweetIndex: {'Resolution': 'Picture in Tweet'}}]) + if item.coordinates: + placeName = 'Tweet Location ' + str(item.id) + if item.place: + placeName = str(item.place.fullName) + if item.place.country: + returnResults.append([{'Country Name': str(item.place.country), + 'Entity Type': 'Country'}, + {tweetIndex: {'Resolution': 'Country in Tweet'}}]) + + returnResults.append([{'Label': placeName, + 'Latitude': str(item.place.latitude), + 'Longitude': str(item.place.longitude), + 'Entity Type': 'GeoCoordinates'}, + {tweetIndex: {'Resolution': 'Coordinates in Tweet'}}]) + if maxResults and index >= maxResults: break