maq/LinkScope_Client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve SNScrape twitter user info extraction. Rework financial entities. Remove redundant resolution 'name' fields from resolutions.

Browse Source
This commit is contained in:
AccentuSoft
2022-02-17 02:49:55 +02:00
parent f859a01a0d
commit 8d843133de
20 changed files with 95 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Core/Entities/Financials.xml
View File

@@ -18,15 +18,14 @@
Currency.svg
</Icon>
</Currency>
<Shares>
<Ticker>
<Attributes>
<Attribute default="Ticker ID" check="String" primary="True">Ticker</Attribute>
<Attribute default="0.0" check="Float" primary="False">Amount</Attribute>
<Attribute default="Ticker ID" check="String" primary="True">Ticker ID</Attribute>
</Attributes>
<Icon>
Share.svg
</Icon>
</Shares>
</Ticker>
<Portfolio>
<Attributes>
<Attribute default="Portfolio Name" check="String" primary="True">Label</Attribute>

12
Modules/CryptoCurrency/BlockChainAPI.py
View File

@@ -49,30 +49,26 @@ class BlockChainAPI:
'Currency Type': 'BTC',
'Notes': 'sent by address',
'Entity Type': 'Currency'},
{uid: {'Resolution': 'sent by address',
'Name': 'sent by address',
{uid: {'Resolution': 'Sent by address',
'Notes': ''}}])
returnResults.append(
[{'Amount': str(int(getReceivedByAddress.text) / 100000000),
'Currency Type': 'BTC',
'Notes': 'received by address',
'Entity Type': 'Currency'},
{uid: {'Resolution': 'received by address',
'Name': 'received by address',
{uid: {'Resolution': 'Received by address',
'Notes': ''}}])
returnResults.append(
[{'Amount': str(int(addressBalance.text) / 100000000),
'Currency Type': 'BTC',
'Notes': 'address balance',
'Entity Type': 'Currency'},
{uid: {'Resolution': 'address balance',
'Name': 'address balance',
{uid: {'Resolution': 'Address balance',
'Notes': ''}}])
returnResults.append(
[{'Date': str(dateCreated),
'Notes': 'first seen date',
'Entity Type': 'Date'},
{uid: {'Resolution': 'first seen date',
'Name': 'first seen date',
{uid: {'Resolution': 'First seen date',
'Notes': ''}}])
return returnResults

18
Modules/EDGAR_&_EFD/CompanyInfo.py
View File

@@ -41,43 +41,37 @@ class CompanyInfo:
exchanges = data['exchanges']
for exchange in exchanges:
returnResults.append([{'Exchange Name': exchange,
'Entity Type': 'Exchanges'},
'Entity Type': 'Exchange'},
{uid: {'Resolution': 'Exchange',
'Name': 'Exchange',
'Notes': ''}}])
tickers = data['tickers']
for ticker in tickers:
returnResults.append([{'Ticker Name': ticker,
'Entity Type': 'Tickers'},
{uid: {'Resolution': 'Exchange',
'Name': 'Exchange',
returnResults.append([{'Ticker ID': ticker,
'Entity Type': 'Ticker'},
{uid: {'Resolution': 'Ticker',
'Notes': ''}}])
if data['insiderTransactionForOwnerExists'] == 1:
returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
else:
returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if data['insiderTransactionForIssuerExists'] == 1:
returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
else:
returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists',
'Entity Type': 'Phrase'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if data['sic'] is not None:
@@ -85,13 +79,11 @@ class CompanyInfo:
'Description': data['sicDescription'],
'Entity Type': 'SIC'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if data['ein'] is not None:
returnResults.append([{'EIN': str(data['ein']),
'Entity Type': 'EIN'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if data['addresses'] is not None:
returnResults.append([{'Street Address': data['addresses']['mailing']['street1'],
@@ -100,7 +92,6 @@ class CompanyInfo:
'Locality': data['addresses']['mailing']['city'],
'Entity Type': 'Address'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']:
@@ -110,6 +101,5 @@ class CompanyInfo:
'Locality': data['addresses']['business']['city'],
'Entity Type': 'Address'},
{uid: {'Resolution': '',
'Name': '',
'Notes': ''}}])
return returnResults

14
Modules/EDGAR_&_EFD/Edgar.xml
View File

@@ -66,16 +66,11 @@
<Attribute default="Unknown" check="String" primary="False">Gross Proceeds Used</Attribute>
</Attributes>
</FormD>
<Tickers>
<Attributes>
<Attribute default="Ticker Name" check="String" primary="True">Ticker Name</Attribute>
</Attributes>
</Tickers>
<Exchanges>
<Exchange>
<Attributes>
<Attribute default="Exchange Name" check="String" primary="True">Exchange Name</Attribute>
</Attributes>
</Exchanges>
</Exchange>
<SIC>
<Attributes>
<Attribute default="0000" check="SIC/NAICS" primary="True">SIC</Attribute>
@@ -102,11 +97,6 @@
<Attribute default="000000000000" check="ISINID" primary="True">ISINID</Attribute>
</Attributes>
</ISINID>
<Shares>
<Attributes>
<Attribute default="0" check="String" primary="True">Amount</Attribute>
</Attributes>
</Shares>
<FormNMFP2>
<Attributes>
<Attribute default="NMFP2 Form Field" check="String" primary="True">Field Name</Attribute>

11
Modules/EDGAR_&_EFD/FramesLookUp.py
View File

@@ -9,7 +9,7 @@ class FramesLookUp:
originTypes = {'Form Field'}
resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Shares'}
resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Phrase'}
parameters = {
'Quarter': {'description': 'Please Ensure that the selected Taxonomy matches the Form Field you typed',
@@ -66,31 +66,26 @@ class FramesLookUp:
returnResults.append([{'Company Name': data['data'][i]['entityName'],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Edgar Company',
'Name': 'Edgar Company',
'Notes': ''}}])
returnResults.append([{'CIK': str(data['data'][i]['cik']).zfill(10),
'Entity Type': 'Edgar ID'},
{index_of_child: {'Resolution': '',
'Name': '',
'Notes': ''}}])
returnResults.append([{'Country Name': data['data'][i]['loc'],
'Entity Type': 'Country'},
{index_of_child: {'Resolution': '',
'Name': '',
'Notes': ''}}])
if unit == 'USD':
returnResults.append([{'Amount': str(data['data'][i]['val']),
'Currency Type': 'USD',
'Entity Type': 'Currency'},
{index_of_child: {'Resolution': 'Form Filed Value',
'Name': 'Edgar ID',
'Notes': ''}}])
elif unit == 'shares':
returnResults.append([{'Amount': str(data['data'][i]['val']),
'Entity Type': 'Shares'},
returnResults.append([{'Phrase': 'Number of Shares: ' + str(data['data'][i]['val']),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Form Filed Value',
'Name': 'Edgar ID',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get10KForms.py
View File

@@ -70,7 +70,6 @@ class Get10KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -90,7 +89,6 @@ class Get10KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get10QForms.py
View File

@@ -69,7 +69,6 @@ class Get10QForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-Q Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -88,7 +87,6 @@ class Get10QForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '10-Q Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

1
Modules/EDGAR_&_EFD/Get13FForms.py
View File

@@ -100,6 +100,5 @@ class Get13FForms:
'Entity Type': 'Form13F'},
{uid: {'Resolution': 'Form13F',
'Name': 'Form13F',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get20FForms.py
View File

@@ -69,7 +69,6 @@ class Get20FForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '20-F Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -89,7 +88,6 @@ class Get20FForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '20-F Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get40FForms.py
View File

@@ -70,7 +70,6 @@ class Get40FForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '40-F Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -90,7 +89,6 @@ class Get40FForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '40-F Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get6KForms.py
View File

@@ -72,7 +72,6 @@ class Get6KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '6-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -92,7 +91,6 @@ class Get6KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '6-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

2
Modules/EDGAR_&_EFD/Get8KForms.py
View File

@@ -72,7 +72,6 @@ class Get8KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '8-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
@@ -92,7 +91,6 @@ class Get8KForms:
'Entity Type': 'Form Field'},
{uid: {'Resolution': '8-K Field',
'Name': 'CIK Edgar ID',
'Notes': ''}}])
return returnResults

1
Modules/EDGAR_&_EFD/GetDForms.py
View File

@@ -114,7 +114,6 @@ class GetDForms:
'Entity Type': 'FormD'},
{uid: {'Resolution': 'D Form',
'Name': 'D Form',
'Notes': ''}}])
people = data['edgarSubmission']['relatedPersonsList']['relatedPersonInfo']

1
Modules/EDGAR_&_EFD/GetN8FForms.py
View File

@@ -70,6 +70,5 @@ class GetN8FForms:
returnResults.append([{'URL': anchor,
'Entity Type': 'Website'},
{uid: {'Resolution': 'N-8F Form',
'Name': 'N-8F Form',
'Notes': ''}}])
return returnResults

6
Modules/EDGAR_&_EFD/GetNMFP2Forms.py
View File

@@ -148,7 +148,6 @@ class GetNMFP2Forms:
'Friday 5': 'NO Value in Daily Measure',
'Entity Type': 'FormNMFP2'},
{uid: {'Resolution': field,
'Name': 'FormNMFP2',
'Notes': ''}}])
else:
returnResults.append([{'Field Name': f'N-MFP2:({field})' + ' '
@@ -165,7 +164,6 @@ class GetNMFP2Forms:
'ns3:fridayWeek5'],
'Entity Type': 'FormNMFP2'},
{uid: {'Resolution': field,
'Name': 'FormNMFP2',
'Notes': ''}}])
classLevelInfo = data['edgarSubmission']['formData']['classLevelInfo']
@@ -189,7 +187,6 @@ class GetNMFP2Forms:
'ns3:fridayWeek5'],
'Entity Type': 'FormNMFP2'},
{index_of_child: {'Resolution': 'Net Asset Per Share',
'Name': 'FormNMFP2',
'Notes': ''}}])
for weekCount in range(1, 6):
returnResults.append(
@@ -262,8 +259,7 @@ class GetNMFP2Forms:
'Ctgry Investments Rprsnts Collateral':
issuer['ctgryInvestmentsRprsntsCollateral'],
'Entity Type': 'Collateral Issuer'},
{index_of_child: {'Resolution': 'Net Asset Per Share',
'Name': 'Collateral Issuer',
{index_of_child: {'Resolution': 'Collateral Issuer',
'Notes': ''}}])
except KeyError:

1
Modules/GitHubRSAFinder/DorkingMethod.py
View File

@@ -61,7 +61,6 @@ class DorkingMethod:
[{'URL': anchor,
'Entity Type': 'Website'},
{uid: {'Resolution': 'RSA Key',
'Name': 'RSA Key',
'Notes': ''}}])
return returnResults

8
Modules/GitHubRSAFinder/OrgSearch_GitAllSecrets.py
View File

@@ -71,14 +71,12 @@ class OrgSearch_GitAllSecrets:
returnResults.append([{'Organisation Name': 'Org or User: ' + userOrg[1],
'Entity Type': 'GitHub Organisation'},
{uid: {'Resolution': 'Tool: repo-supervisor',
'Name': 'Tool: repo-supervisor',
'Notes': ''}}])
child_of_child = len(returnResults)
returnResults.append([{'Repository Name': userOrg[3],
'Entity Type': 'GitHub Repository'},
{index_of_child: {'Resolution': 'Repository of Organisation',
'Name': 'Repository of Organisation',
'Notes': ''}}])
index += 1
@@ -94,7 +92,6 @@ class OrgSearch_GitAllSecrets:
'Notes': ansi_escape.sub('', hogSecret[hogIndex]),
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'GitHub Secret',
'Name': 'GitHub Secret',
'Notes': ''}}])
# print(ansi_escape.sub('', hogSecret[hogIndex]))
hogIndex += 1
@@ -102,19 +99,16 @@ class OrgSearch_GitAllSecrets:
returnResults.append([{'Hash Value': ansi_escape.sub('', line),
'Entity Type': 'Hash'},
{childOfChild: {'Resolution': 'Tool: truffleHog',
'Name': 'GitHub Hash',
'Notes': ''}}])
elif 'Filepath' in line:
returnResults.append([{'Filepath': ansi_escape.sub('', line),
'Entity Type': 'GitHub FilePath'},
{childOfChild: {'Resolution': 'GitHub FilePath',
'Name': 'GitHub FilePath',
'Notes': ''}}])
elif 'Branch' in line:
returnResults.append([{'Branch': ansi_escape.sub('', line),
'Entity Type': 'GitHub Branch'},
{childOfChild: {'Resolution': 'GitHub Branch',
'Name': 'GitHub Branch',
'Notes': ''}}])
for result in data:
@@ -125,14 +119,12 @@ class OrgSearch_GitAllSecrets:
returnResults.append([{'Filepath': secret.get('filepath'),
'Entity Type': 'GitHub FilePath'},
{child_of_child: {'Resolution': 'GitHub FilePath',
'Name': 'GitHub FilePath',
'Notes': ''}}])
for scrt in secrets:
returnResults.append([{'Secret': scrt,
'Entity Type': 'GitHub Secret'},
{child_child: {'Resolution': 'GitHub Secret',
'Name': 'GitHub Secret',
'Notes': ''}}])
return returnResults

46
Modules/Pulsedive_API/IndicatorsInfo.py
View File

@@ -61,23 +61,23 @@ class IndicatorsInfo:
returnResults.append([{'Phrase': indicator['description'],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Pulsedive Indicator Scan',
'Name': 'Pulsedive Indicator Scan', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phrase': 'risk:' + indicator['risk'],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Risk',
'Name': 'Risk', 'Notes': ''}}])
'Notes': ''}}])
for port in data["attributes"]["port"]:
returnResults.append([{'Port': search_term + ':' + str(port),
'Entity Type': 'Port'},
{uid: {'Resolution': 'Ports Open',
'Name': 'Ports Open', 'Notes': ''}}])
'Notes': ''}}])
for technology in data["attributes"]["technology"]:
returnResults.append([{'Phrase': technology,
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Technologies Used',
'Name': 'Technologies Used', 'Notes': ''}}])
'Notes': ''}}])
index_of_child = len(returnResults)
@@ -87,17 +87,17 @@ class IndicatorsInfo:
'Postal Code': data["properties"]["geo"]['zip'],
'Entity Type': 'Address'},
{uid: {'Resolution': 'Address',
'Name': 'Address', 'Notes': ''}}])
'Notes': ''}}])
elif data["properties"]["geo"].get('country') is not None:
returnResults.append([{'Country Name': data["properties"]["geo"]['country'],
'Entity Type': 'Country'},
{index_of_child: {'Resolution': 'Country',
'Name': 'Country', 'Notes': ''}}])
'Notes': ''}}])
elif data["properties"]["geo"].get('city') is not None:
returnResults.append([{'City Name': data["properties"]["geo"]['city'],
'Entity Type': 'City'},
{index_of_child: {'Resolution': 'City',
'Name': 'City', 'Notes': ''}}])
'Notes': ''}}])
dns_list = data["properties"]["dns"]
for key in dns_list.keys():
@@ -112,12 +112,12 @@ class IndicatorsInfo:
returnResults.append([{'Domain Name': value,
'Entity Type': 'Domain'},
{uid: {'Resolution': 'MX Record',
'Name': 'MX Record', 'Notes': ''}}])
'Notes': ''}}])
elif dns_type[i] == "a":
returnResults.append([{'IP Address': data["properties"]["dns"]["a"],
'Entity Type': 'IP Address'},
{index_of_child: {'Resolution': 'A Record',
'Name': 'A Record', 'Notes': ''}}])
'Notes': ''}}])
elif dns_type[i] == "txt":
for value in data["properties"]["dns"]['txt']:
@@ -125,34 +125,34 @@ class IndicatorsInfo:
'Entity Type': 'Phrase',
'Notes': value},
{uid: {'Resolution': 'TXT Record',
'Name': 'TXT Record', 'Notes': ''}}])
'Notes': ''}}])
elif dns_type[i] == "soa":
returnResults.append([{'Domain Name': data["properties"]["dns"]['rname'],
'Entity Type': 'Domain'},
{uid: {'Resolution': 'Rname Record',
'Name': 'Rname Record', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Domain Name': data["properties"]["dns"]['soa'],
'Entity Type': 'Domain'},
{index_of_child: {'Resolution': 'SOA Record',
'Name': 'SOA Record', 'Notes': ''}}])
'Notes': ''}}])
elif dns_type[i] == "ns":
for value in data["properties"]["dns"]['ns']:
returnResults.append([{'Domain Name': value,
'Entity Type': 'Domain'},
{uid: {'Resolution': 'NS Record',
'Name': 'NS Record', 'Notes': ''}}])
'Notes': ''}}])
if data["properties"].get("ssl") is not None:
returnResults.append([{'Date': data["properties"]["ssl"]['expires'],
'Notes': 'expiration date of ssl certificate',
'Entity Type': 'Date'},
{uid: {'Resolution': 'expiration date of ssl certificate',
'Name': 'expiration date of ssl certificate', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Company Name': data["properties"]["ssl"]['org'],
'Notes': 'issuer of certificate',
'Entity Type': 'Company'},
{uid: {'Resolution': 'issuer of certificate',
'Name': 'issuer of certificate', 'Notes': ''}}])
'Notes': ''}}])
for key in data["properties"]["whois"].keys():
whois_type.append(key)
@@ -164,27 +164,27 @@ class IndicatorsInfo:
returnResults.append([{'Company Name': field[kind],
'Entity Type': 'Company'},
{uid: {'Resolution': 'Registrar Company',
'Name': 'Registrar Company', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phrase': 'registry domain id' + ":" + field["registry domain id"],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Registry Domain ID',
'Name': 'registry domain id', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phrase': "registrar iana id" + ":" + field["registrar iana id"],
'Entity Type': 'Phrase'},
{index_of_child: {'Resolution': 'Registrar IANA ID',
'Name': 'registrar iana id', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Email Address': field["registrar abuse contact email"],
'Entity Type': 'Email Address'},
{index_of_child: {'Resolution': 'Registrar abuse contact email',
'Name': 'Registrar abuse contact email', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phone Number': field["registrar abuse contact phone"],
'Entity Type': 'Phone Number'},
{index_of_child: {'Resolution': 'Registrar abuse contact phone',
'Name': 'Registrar abuse contact phone', 'Notes': ''}}])
'Notes': ''}}])
elif kind == 'admin organization':
index_of_child = len(returnResults)
@@ -192,16 +192,16 @@ class IndicatorsInfo:
'Notes': 'admin organization',
'Entity Type': 'Company'},
{uid: {'Resolution': 'Admin Organisation',
'Name': 'Admin Organisation', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Email Address': field['admin email'],
'Notes': 'admin email',
'Entity Type': 'Email Address'},
{index_of_child: {'Resolution': 'Admin Email Address',
'Name': 'Admin Email Address', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phone Number': field['admin phone'],
'Notes': 'admin phone',
'Entity Type': 'Phone Number'},
{index_of_child: {'Resolution': 'Admin Phone Number',
'Name': 'Admin Phone Number', 'Notes': ''}}])
'Notes': ''}}])
return returnResults

10
Modules/Pulsedive_API/ThreatInfo.py
View File

@@ -60,11 +60,11 @@ class ThreatInfo:
returnResults.append([{'Phrase': 'category: ' + data["category"],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Category',
'Name': 'Category', 'Notes': ''}}])
'Notes': ''}}])
returnResults.append([{'Phrase': 'risk: ' + data["risk"],
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Risk',
'Name': 'Risk', 'Notes': ''}}])
'Notes': ''}}])
if linkNumbers == 0:
linkNumbers = int(len(data["news"]))
@@ -76,7 +76,7 @@ class ThreatInfo:
'Date Created': data["news"][i]['stamp'].replace(' ', 'T'),
'Entity Type': 'Website'},
{uid: {'Resolution': 'Related Article',
'Name': 'Related Article', 'Notes': ''}}])
'Notes': ''}}])
if not data["summary"]["attributes"]:
continue
@@ -85,12 +85,12 @@ class ThreatInfo:
returnResults.append([{'Phrase': key,
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Technologies Used',
'Name': 'Technologies Used', 'Notes': ''}}])
'Notes': ''}}])
for key in data["summary"]["attributes"]["protocol"].keys():
returnResults.append([{'Phrase': key,
'Entity Type': 'Phrase'},
{uid: {'Resolution': 'Protocols Used',
'Name': 'Protocols Used', 'Notes': ''}}])
'Notes': ''}}])
return returnResults

52
Modules/SNScrape/TwitterUser.py
View File

@@ -144,7 +144,7 @@ class TwitterUser:
'User URL': item.user.url,
'Verified': str(item.user.verified),
'Display Name': item.user.displayname,
'Location': item.user.location,
'Location': str(item.user.location),
'Description': item.user.description,
'Protected': str(item.user.protected),
'Followers': str(item.user.followersCount),
@@ -157,6 +157,7 @@ class TwitterUser:
'Icon': iconByteArrayFin, # If None, it will have the pic for Twitter User
'Date Created': item.user.created.isoformat()},
{uid: {'Resolution': 'Twitter User'}}])
tweetIndex = len(returnResults)
returnResults.append([{'Tweet ID': str(item.id),
'Tweet URL': item.url,
'Replies': str(item.replyCount),
@@ -167,8 +168,55 @@ class TwitterUser:
'Place': str(item.place),
'Entity Type': 'Tweet',
'Date Created': item.date.isoformat(),
'Notes': item.content},
'Notes': str(item.content)},
{childIndex: {'Resolution': 'Tweet'}}])
if item.outlinks:
for link in item.outlinks:
returnResults.append([{'URL': link,
'Entity Type': 'Website'},
{tweetIndex: {'Resolution': 'External Link in Tweet'}}])
if item.cashtags:
for cashtag in item.cashtags:
returnResults.append([{'Ticker ID': cashtag,
'Entity Type': 'Ticker'},
{tweetIndex: {'Resolution': 'Cashtag in Tweet'}}])
if item.hashtags:
for hashtag in item.hashtags:
returnResults.append([{'Phrase': hashtag,
'Entity Type': 'Phrase'},
{tweetIndex: {'Resolution': 'Hashtag in Tweet'}}])
if item.media:
for mediaItem in item.media:
if mediaItem.variants:
maxBitrate = 0
bestVariant = None
for variant in mediaItem.variants:
if variant.bitrate and variant.bitrate > maxBitrate:
maxBitrate = variant.bitrate
bestVariant = variant
if bestVariant:
returnResults.append([{'URL': str(bestVariant.url),
'Entity Type': 'Website'},
{tweetIndex: {'Resolution': 'Video in Tweet'}}])
else:
returnResults.append([{'URL': str(mediaItem.fullUrl),
'Entity Type': 'Website'},
{tweetIndex: {'Resolution': 'Picture in Tweet'}}])
if item.coordinates:
placeName = 'Tweet Location ' + str(item.id)
if item.place:
placeName = str(item.place.fullName)
if item.place.country:
returnResults.append([{'Country Name': str(item.place.country),
'Entity Type': 'Country'},
{tweetIndex: {'Resolution': 'Country in Tweet'}}])
returnResults.append([{'Label': placeName,
'Latitude': str(item.place.latitude),
'Longitude': str(item.place.longitude),
'Entity Type': 'GeoCoordinates'},
{tweetIndex: {'Resolution': 'Coordinates in Tweet'}}])
if maxResults and index >= maxResults:
break