Fix bug with Social_Analyzer.py - need to include all contents of 'tld' module.

2022-04-30 19:41:04 +03:00
parent 7c7288d949
commit d05f13138f
4 changed files with 33 additions and 32 deletions
--- a/Core/Interface/MenuBar.py
+++ b/Core/Interface/MenuBar.py
@@ -895,11 +895,11 @@ class MenuBar(QtWidgets.QMenuBar):
    def downloadFiles(self) -> None:
        self.parent().downloadFiles()

-    def manageCollectors(self) -> None:
-        with self.parent().serverCollectorsLock:
-            currentCollectors = dict(self.parent().collectors)
-        collectorsDialog = CollectorsDialog(currentCollectors if self.parent().FCOM.isConnected() else None)
-        collectorsDialog.exec()
+    #def manageCollectors(self) -> None:
+    #    with self.parent().serverCollectorsLock:
+    #        currentCollectors = dict(self.parent().collectors)
+    #    collectorsDialog = CollectorsDialog(currentCollectors if self.parent().FCOM.isConnected() else None)
+    #    collectorsDialog.exec()

    def selectAllNodes(self) -> None:
        self.parent().centralWidget().tabbedPane.getCurrentScene().selectAllNodes()
--- a/Modules/Username_Crosscheck/Social_Analyzer_Detector.py
+++ b/Modules/Username_Crosscheck/Social_Analyzer_Detector.py
@@ -5,32 +5,32 @@ Note: Creates log files in temp folders in /tmp. Files are only readable by the
 """


-class Social_Analyzer:
+class Social_Analyzer_Detector:
    name = "Social Analyzer"
    category = "Online Identity"
    description = "Find potentially connected social media accounts."
    originTypes = {'Phrase', 'Social Media Handle'}
    resultTypes = {'Social Media Handle'}
-    parameters = {'websites': {'description': 'Enter the domain names of the social media websites'
-                                              ' to check or "all" (no quotes) to check all available websites.'
-                                              ' Note that some of the results could be false positives.',
+    parameters = {'websites': {'description': 'Enter the domain names of the social media websites '
+                                              'to check, separated with spaces, or "all" (no quotes) '
+                                              'to check all available websites.\n'
+                                              'E.g.: "github.com instagram.com" (no quotes).',
                               'type': 'String',
                               'default': 'all',
-                               'value': 'e.g: Facebook.com, Instagram.com etc...'}}
+                               'value': 'Example input: facebook.com instagram.com'}}

    def resolution(self, entityJsonList, parameters):
        import requests
        from importlib import import_module
        from requests.exceptions import RequestException
-        import time
        import random
        import string
        import re
        commentsRegex = re.compile('<!--.*?-->', re.DOTALL)
-        websites = parameters['websites']
+        websites = parameters['websites'].lower()

        headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
        }

        def resolutionHelper(original_url):
@@ -39,8 +39,7 @@ class Social_Analyzer:
            certificate verification is disabled.
            """
            try:
-                originalUsernameRegex = re.escape(social_field)
-                originalUsernameRegex2 = re.compile(originalUsernameRegex, re.IGNORECASE)
+                originalUsernameRegex = re.compile(re.escape(social_field), re.IGNORECASE)
                firstResponse = requests.get(original_url, verify=False, timeout=30, allow_redirects=False,  # nosec
                                             headers=headers)
                if firstResponse.status_code >= 300:
@@ -51,7 +50,7 @@ class Social_Analyzer:
                    usernameRegex = re.compile(social_field, re.IGNORECASE)
                    r = requests.get(original_url, timeout=30, verify=False, headers=headers)  # nosec
                    originalContent = r.text
-                    if len(originalUsernameRegex2.findall(originalContent)) == 0:
+                    if len(originalUsernameRegex.findall(originalContent)) == 0:
                        # False Positive
                        return False
                    modified_url = original_url.replace(social_field, modifiedUsername)
@@ -78,7 +77,13 @@ class Social_Analyzer:
            # Have to keep re-importing so that it works for entities beyond the first.
            SocialAnalyzer = import_module("social-analyzer").SocialAnalyzer(silent=True)
            uid = entity['uid']
-            social_field = entity[list(entity)[1]].strip().lower()
+            if entity['Entity Type'] == 'Phrase':
+                social_field = entity['Phrase']
+            elif entity['Entity Type'] == 'Social Media Handle':
+                social_field = entity['User Name']
+            else:
+                continue
+            social_field = social_field.strip().lower()
            results = SocialAnalyzer.run_as_object(
                username=social_field, silent=True, output="json", filter='good', metadata=False, logs_dir='',
                websites=websites, mode='fast', timeout=10, profiles='detected')
@@ -89,14 +94,7 @@ class Social_Analyzer:
            if len(results) != 0:
                for link in results['detected']:
                    url = link['link']
-                    count = 0
                    helperResult = resolutionHelper(url)
-                    while helperResult is None:
-                        time.sleep(5)
-                        count += 1
-                        if count == 3:
-                            break
-                        helperResult = resolutionHelper(url)
-                    if count != 3 and helperResult is not False:
+                    if isinstance(helperResult, list):
                        return_result.append(helperResult)
        return return_result
--- a/build.sh
+++ b/build.sh
@@ -7,9 +7,11 @@ python3.9 -m venv buildEnv

 source buildEnv/bin/activate

-python3.9 -m pip install --upgrade pip pyinstaller
+python3.9 -m pip install --upgrade wheel pip pyinstaller

-python3.9 -m pip install --upgrade -r requirements.txt
+python3.9 -m pip cache purge
+
+python3.9 -m pip install -r requirements.txt

 PLAYWRIGHT_BROWSERS_PATH=0 python3.9 -m playwright install

@@ -21,7 +23,7 @@ python3.9 -m PyInstaller --clean --icon="./Icon.ico" --noconsole --noconfirm --o
 --collect-all "docker" --collect-all "exif" --collect-all "pycountry" --collect-all "tldextract" \
 --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" --hidden-import "pandas" \
 --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" --collect-all "Wappalyzer" \
--collect-all "email_validator" --add-data "./buildEnv/lib/python3.9/site-packages/social-analyzer:social-analyzer/" \
+--collect-all "email_validator" --collect-all "social-analyzer" --collect-all "tld" \
 --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" \
 --hidden-import "defusedxml" --hidden-import "cchardet" --hidden-import "ipwhois" --hidden-import "xmltodict" \
 --hidden-import "dateutil" --hidden-import "urllib3" --hidden-import "logging" --hidden-import "holehe" \
--- a/buildWin.bat
+++ b/buildWin.bat
@@ -7,9 +7,11 @@ python -m venv buildEnv

 call buildEnv\Scripts\activate.bat

-python -m pip install --upgrade pip pyinstaller
+python -m pip install --upgrade wheel pip pyinstaller

-python -m pip install --upgrade -r requirements.txt
+python -m pip cache purge
+
+python -m pip install -r requirements.txt

 set PLAYWRIGHT_BROWSERS_PATH=0
 playwright install
@@ -25,8 +27,7 @@ python -m PyInstaller --clean --icon="Icon.ico" --noconsole --noconfirm --onedir
 --collect-all "docker" --collect-all "exif" --collect-all "dns" --collect-all "pycountry" ^
 --collect-all "tldextract" --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" ^
 --hidden-import "pandas" --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" ^
--collect-all "Wappalyzer" --collect-all "email_validator" ^
--add-data "C:\Users\IEUser\AppData\Roaming\Python\Python39\site-packages\social-analyzer;social-analyzer" ^
+--collect-all "Wappalyzer" --collect-all "email_validator" --collect-all "social-analyzer" --collect-all "tld" ^
 --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" ^
 --hidden-import "logging" --hidden-import "defusedxml" --hidden-import "dateutil" --hidden-import "xmltodict" ^
 --hidden-import "urllib3" --hidden-import "cchardet" --hidden-import "ipwhois" --hidden-import "holehe" ^