Change 'SNScrape' module name to 'Social Media'. Added Twayback resolution to get deleted tweets to the module.

Minor change in link name for Reddit.py in Reddit module.

Modules/Reddit/Reddit.py

@@ -80,7 +80,7 @@ class Reddit:
                                           {index_of_child: {'Resolution': resolution_name,
                                                             'Notes': ''}}])
 
-                    comment_resolution = 'Reddit Comment Hashed'
+                    comment_resolution = 'Reddit Comment Hash'
                     comment = hashlib.md5(value.get('body', 'N/A').encode())  # nosec
                     comment = hexlify(comment.digest()).decode()
                     return_result.append([{'Comment': comment,

Modules/SNScrape/requirements.txt

@@ -1,2 +0,0 @@
-snscrape
-pytz

Modules/SocialMedia/Twayback.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+"""
+Credit:
+https://github.com/Mennaruuk/twayback
+"""
+
+
+class Twayback:
+    name = "Get Deleted Tweets"
+    category = "Online Identity"
+    description = "Get any deleted tweets belonging to the twitter user specified."
+    originTypes = {'Social Media Handle', 'Phrase', 'Twitter User'}
+    resultTypes = {'Website'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import bs4
+        import re
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        from time import sleep
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            account_name = entity[list(entity)[1]]
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'}
+            futures = []
+
+            wayback_cdx_url = f"https://web.archive.org/cdx/search/cdx?url=twitter.com/{account_name}/status" \
+                              f"&matchType=prefix&filter=statuscode:200&mimetype:text/html&from=&to="
+            cdx_page_text = requests.get(wayback_cdx_url).text
+
+            if len(re.findall(r'Blocked', cdx_page_text)) != 0:
+                return f"Sorry, no deleted Tweets can be retrieved for {account_name}.\n" \
+                       f"This is because the Wayback Machine excludes Tweets for this handle."
+
+            # Capitalization does not matter for twitter links. Url parameters after '?' do not matter either.
+            tweet_id_and_url_dict = {line.split()[2].lower().split('?')[0]: line.split()[1] for line in
+                                     cdx_page_text.splitlines()}
+
+            with FuturesSession(max_workers=10) as session:
+                for twitter_url in tweet_id_and_url_dict:
+                    futures.append(session.get(twitter_url, headers=headers, timeout=30, allow_redirects=False))
+            missing_tweets = {}
+
+            for future in as_completed(futures):
+                # Cannot display progress or log stuff here, as this is done in other threads.
+                page_response = future.result()
+                if page_response.status_code == 404:
+                    split_once = page_response.url.split('/status/')[-1]
+                    split_fin = re.split(r'\D', split_once)[0]
+                    missing_tweets[page_response.url] = split_fin
+
+            wayback_url_list = {}
+
+            for url, number in missing_tweets.items():
+                wayback_url_list[number] = f"https://web.archive.org/web/{number}/{url}"
+
+            deleted_tweets_futures_retry = []
+
+            futures_list = []
+            regex = re.compile('.*TweetTextSize TweetTextSize--jumbo.*')
+
+            with FuturesSession(max_workers=10) as session:
+                for number, url in wayback_url_list.items():
+                    futures_list.append(session.get(url, headers=headers, timeout=30))
+            for future in as_completed(futures_list):
+                result = None
+                try:
+                    result = future.result()
+                    tweet = bs4.BeautifulSoup(result.content, "lxml").find("p", {"class": regex}).getText()
+                    returnResults.append([{'URL': result.url,
+                                           'Entity Type': 'Website',
+                                           'Notes': tweet},
+                                          {uid: {'Resolution': 'Deleted Tweet'}}])
+                except AttributeError:
+                    pass
+                except ConnectionError:
+                    if result is not None:
+                        deleted_tweets_futures_retry.append(result.url)
+
+            # Second try, if things go wrong.
+            if len(deleted_tweets_futures_retry) > 0:
+                sleep(10)
+                futures_list = []
+                with FuturesSession(max_workers=10) as session:
+                    for url in deleted_tweets_futures_retry:
+                        futures_list.append(session.get(url))
+                for future in as_completed(futures_list):
+                    try:
+                        result = future.result()
+                        tweet = bs4.BeautifulSoup(result.content, "lxml").find("p", {"class": regex}).getText()
+                        returnResults.append([{'URL': result.url,
+                                               'Entity Type': 'Website',
+                                               'Notes': tweet},
+                                              {uid: {'Resolution': 'Deleted Tweet'}}])
+                    except AttributeError:
+                        pass
+                    except ConnectionError:
+                        pass
+
+        return returnResults

Modules/SocialMedia/requirements.txt

@@ -0,0 +1,3 @@
+snscrape
+pytz
+requests_futures