Refactored JSCodeExtractor.py to use playwright.

Changed an if statement in Entity.py to be a one-liner.

Core/Interface/Entity.py

@@ -263,10 +263,7 @@ class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
         currentStartPos = self.myStartItem.pos()
         currentEndPos = self.myEndItem.pos()
 
-        if self.isSelected():
-            self.myColor = self.colorSelected
-        else:
-            self.myColor = self.colorDefault
+        self.myColor = self.colorSelected if self.isSelected() else self.colorDefault
 
         if currentEndPos == self.oldEndPos and currentStartPos == self.oldStartPos:
             myPen = QtGui.QPen(self.myColor)

Core/Resolutions/Core/JSCodeExtractor.py

@@ -8,61 +8,58 @@ class JSCodeExtractor:
     # A string that describes this resolution.
     description = "Returns Nodes of 'ca-pub', 'ua' and 'gtm' tracking codes for domains."
 
-    originTypes = {'Domain'}
+    originTypes = {'Website'}
 
     resultTypes = {'Phrase'}
 
     parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-
-        def GetTrackingCodes(domain):
-            import re
-            from seleniumwire import webdriver
-
-            Identifiers = []
-            uaIDs = []
-            uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
-            pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
-            gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
-            gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
-            trackingSites = ['client=ca-pub', 'id=UA', 'id=GTM', 'id=G']
-
-            fireFoxOptions = webdriver.FirefoxOptions()
-            fireFoxOptions.headless = True
-            driver = webdriver.Firefox(options=fireFoxOptions)
-            # Access requests via the `requests` attribute
-            driver.get(domain)
-            for request in driver.requests:
-                requestUrl = str(request.url)
-                if [ele for ele in trackingSites if (ele in requestUrl)]:
-                    for uaRegexMatch in uaRegex.findall(str(requestUrl)):
-                        uaIDs.append(uaRegexMatch)
-                    for pubRegexMatch in pubRegex.findall(str(requestUrl)):
-                        Identifiers.append(pubRegexMatch)
-                    for gtmRegexMatch in gtmRegex.findall(str(requestUrl)):
-                        Identifiers.append(gtmRegexMatch)
-                    for gRegexMatch in gRegex.findall(str(requestUrl)):
-                        Identifiers.append(gRegexMatch)
-            for ua in list(set(uaIDs)):
-                Identifiers.append(''.join(ua.split('-')[:-1]))
-            driver.quit()
-            return list(set(Identifiers))
-
+        from playwright.sync_api import sync_playwright
+        import re
         returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            if entity[list(entity)[1]].startswith('http://') or entity[list(entity)[1]].startswith('https://'):
-                url = entity[list(entity)[1]]
-            else:
-                url = 'http://' + entity[list(entity)[1]]
-            IDs = GetTrackingCodes(url)
 
-            for i in IDs:
-                returnResults.append([{'Phrase': i,
+        uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
+        pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
+        gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
+        gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
+        trackingSites = ['client=ca-pub', 'id=UA', 'id=GTM', 'id=G']
+
+        def GetTrackingCodes(pageUid, requestUrl) -> None:
+            trackingCodes = []
+            if [ele for ele in trackingSites if (ele in requestUrl)]:
+                for uaRegexMatch in uaRegex.findall(str(requestUrl)):
+                    trackingCodes.append(uaRegexMatch)
+                for pubRegexMatch in pubRegex.findall(str(requestUrl)):
+                    trackingCodes.append(pubRegexMatch)
+                for gtmRegexMatch in gtmRegex.findall(str(requestUrl)):
+                    trackingCodes.append(gtmRegexMatch)
+                for gRegexMatch in gRegex.findall(str(requestUrl)):
+                    trackingCodes.append(gRegexMatch)
+
+            for trackingCode in trackingCodes:
+                returnResults.append([{'Phrase': trackingCode,
                                        'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Tracker ID',
-                                             'Name': 'Tracker ID',
-                                             'Notes': ''}}])
+                                      {pageUid: {'Resolution': 'Tracking Code',
+                                                 'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+
+                page = context.new_page()
+                # Subscribe to "request" events.
+                page.on("request", lambda request: GetTrackingCodes(uid, request.url))
+                page.goto(url)
 
         return returnResults