Update build script for Windows.

Dockbar Two now has 'Notes' tab that allows investigators to take canvas-specific notes.
Allows Markdown formatting.

Markdown editor now used for editing & displaying notes for entities and links as well.

Breaking change: Canvases file renamed from canvasTabs.lscanvas to CanvasTabs.lscanvas.

Core/Entities/Financials.xml

@@ -34,4 +34,13 @@
             Portfolio.svg
         </Icon>
     </Portfolio>
+    <Crypto_Wallet>
+        <Attributes>
+          <Attribute default="Wallet Address" check="String" primary="True">Wallet Address</Attribute>
+          <Attribute default="Bitcoin" check="String" primary="False">Currency Name</Attribute>
+        </Attributes>
+        <Icon>
+          CryptoWallet.svg
+        </Icon>
+    </Crypto_Wallet>
 </Financials>

Core/Entities/Individuals.xml

@@ -1,10 +1,11 @@
 <Individuals>
   <Person>
     <Attributes>
-      <Attribute default="John Smith" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
     </Attributes>
     <Icon>
       Person.svg
@@ -12,13 +13,14 @@
   </Person>
   <Politically_Exposed_Person>
     <Attributes>
-      <Attribute default="John Smith" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
     </Attributes>
     <Icon>
-      Person.svg
+      PEP.svg
     </Icon>
   </Politically_Exposed_Person>
   <Phrase>

Core/Entities/Infrastructure.xml

@@ -9,7 +9,7 @@
   </Operating_System>
   <Network>
     <Attributes>
-      <Attribute default="127:0:0:1" check="String" primary="True">IP Address</Attribute>
+      <Attribute default="127.0.0.0" check="String" primary="True">IP Address</Attribute>
       <Attribute default="Unknown" check="String" primary="False">Range</Attribute>
     </Attributes>
     <Icon>
@@ -24,6 +24,14 @@
       Website.svg
     </Icon>
   </Website>
+  <Onion_Website>
+    <Attributes>
+      <Attribute default="http://exampleexampleexampleexampleexampleexampleexampleexample.onion" check="Onion" primary="True">Onion URL</Attribute>
+    </Attributes>
+    <Icon>
+      OnionWebsite.svg
+    </Icon>
+  </Onion_Website>
   <Domain>
     <Attributes>
       <Attribute default="example.com" check="Domain" primary="True">Domain Name</Attribute>

Core/Entities/Meta.xml

@@ -2,7 +2,7 @@
   <EntityGroup>
     <Attributes>
       <Attribute default="Entity Group" check="String" primary="True">Group Name</Attribute>
-      <Attribute default="[]" check="String" primary="False">Child UIDs</Attribute>
+      <Attribute default="[]" check="None" primary="False">Child UIDs</Attribute>
     </Attributes>
     <Icon>
       GroupNode.svg

Core/Entities/Social Media.xml

@@ -2,7 +2,6 @@
     <Social_Media_Group>
         <Attributes>
             <Attribute default="Guest Group" check="String" primary="True">Group Name</Attribute>
-            <Attribute default="Unknown / Multiple" check="String" primary="False">Social Media Site</Attribute>
         </Attributes>
         <Icon>
             Social_Media_Group.svg

Core/Entities/Software.xml

@@ -16,4 +16,13 @@
             Hash.svg
         </Icon>
     </Hash>
+    <CVE>
+        <Attributes>
+            <Attribute default="Unassigned CVE" check="String" primary="True">CVE</Attribute>
+            <Attribute default="0.0" check="Float" primary="False">CVSS Score</Attribute>
+        </Attributes>
+        <Icon>
+            CVE.svg
+        </Icon>
+    </CVE>
 </Software>

Core/Entities/Vehicles.xml

@@ -0,0 +1,24 @@
+<Vehicles>
+    <Vessel>
+        <Attributes>
+            <Attribute default="IMO 0" check="String" primary="True">Vessel Registration</Attribute>
+            <Attribute default="Vessel" check="String" primary="False">Vessel Name</Attribute>
+            <Attribute default="000000" check="String" primary="False">Vessel CallSign</Attribute>
+            <Attribute default="XX" check="String" primary="False">Vessel Flag</Attribute>
+        </Attributes>
+        <Icon>
+            Vessel.svg
+        </Icon>
+    </Vessel>
+    <Aircraft>
+        <Attributes>
+            <Attribute default="0000" check="String" primary="True">Aircraft Serial Number</Attribute>
+            <Attribute default="00-000" check="String" primary="False">Aircraft Registration</Attribute>
+            <Attribute default="Aircraft" check="String" primary="False">Aircraft Name</Attribute>
+            <Attribute default="Aircraft Model" check="String" primary="False">Aircraft Model</Attribute>
+        </Attributes>
+        <Icon>
+            Aircraft.svg
+        </Icon>
+    </Aircraft>
+</Vehicles>

Core/EntityDB.py

@@ -51,7 +51,7 @@ class EntitiesDB:
 
     def resetTimeline(self) -> None:
         """
-        Reset the timeline on dockBarThree to reflect the current state of the entities database.
+        Reset the timeline on dockBarThree to reflect the current state of the database.
         """
         self.dbLock.acquire()
         if self.database is not None:
@@ -60,7 +60,7 @@ class EntitiesDB:
 
     def updateTimeline(self, node, added: bool, updateGraph: bool = True) -> None:
         """
-        Update the timeline on dockBarThree to reflect the newest change of the entities database.
+        Update the timeline on dockBarThree to reflect the newest change of the database.
         """
         self.dbLock.acquire()
         self.mainWindow.updateTimeline(node, added, updateGraph)
@@ -87,10 +87,6 @@ class EntitiesDB:
     def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True) -> Union[dict, None]:
         """
         Adds the entity represented by the json dictionary to the database.
-        :param updateTimeline:
-        :param entJson:
-        :param fromServer:
-        :return:
         """
         self.dbLock.acquire()
         returnValue = None
@@ -107,7 +103,7 @@ class EntitiesDB:
         if entity is None:
             self.dbLock.release()
             return returnValue
-        # Use uid as key. Code is holdover from time where primary field == uid.
+        # Use uid as key. Code is holdover from the time when primary field == uid.
         self.database.add_node(entity['uid'], **entity)
         returnValue = entity
         if exists:
@@ -126,11 +122,11 @@ class EntitiesDB:
 
         return returnValue
 
-    def addEntities(self, entsJsonList: Union[list, set, tuple], fromServer: bool = False) -> list:
+    def addEntities(self, entitiesJsonList: Union[list, set, tuple], fromServer: bool = False) -> list:
         self.dbLock.acquire()
         returnValue = []
 
-        for entJson in entsJsonList:
+        for entJson in entitiesJsonList:
             # Check if we're overwriting an existing entity
             exists = None
             if entJson.get('uid') is not None:
@@ -142,7 +138,7 @@ class EntitiesDB:
 
             if entity is None:
                 continue
-            # Use uid as key. Code is holdover from time where primary field == uid.
+            # Use uid as key. Code is holdover from the time when primary field == uid.
             self.database.add_node(entity['uid'], **entity)
             returnValue.append(entity)
             if exists:

Core/FrontendCommunicationsHandler.py

@@ -49,6 +49,9 @@ class CommunicationsHandler(QtCore.QObject):
     connected_to_server_listener = QtCore.Signal(str)
     receive_question_answer = QtCore.Signal(dict)
     receive_chat_message = QtCore.Signal(str)
+    receive_collectors_signal = QtCore.Signal(dict, dict)
+    receive_start_collector_signal = QtCore.Signal(str, str, str, list, dict)
+    receive_collector_result_signal = QtCore.Signal(str, str, str, list)
     receive_resolutions_signal = QtCore.Signal(dict)
     receive_completed_resolution_result_signal = QtCore.Signal(str, list)
     receive_completed_resolution_string_result_signal = QtCore.Signal(str, str)
@@ -84,6 +87,9 @@ class CommunicationsHandler(QtCore.QObject):
         self.connected_to_server_listener.connect(self.mainWindow.connectedToServerListener)
         self.receive_question_answer.connect(self.mainWindow.questionAnswerListener)
         self.receive_chat_message.connect(self.mainWindow.receiveChatMessage)
+        self.receive_collectors_signal.connect(self.mainWindow.addCollectorsFromServerListener)
+        self.receive_start_collector_signal.connect(self.mainWindow.startNewCollectorListener)
+        self.receive_collector_result_signal.connect(self.mainWindow.receiveCollectorResultListener)
         self.receive_resolutions_signal.connect(self.mainWindow.addResolutionsFromServerListener)
         self.receive_completed_resolution_result_signal.connect(self.mainWindow.resolutionSignalListener)
         self.receive_completed_resolution_string_result_signal.connect(self.mainWindow.resolutionSignalListener)
@@ -203,7 +209,7 @@ class CommunicationsHandler(QtCore.QObject):
         encryptor = self.cipher.encryptor()
         padNeeded = 16 - (len(bytesObject) % 16)
         message = encryptor.update(b'a' * padNeeded + bytesObject) + encryptor.finalize()
-        return b64encode(message) + b'\x00\x00\x00'
+        return b64encode(message)
 
     def decryptTransmission(self, bytesObject) -> Union[bytes, None]:
         decrypter = self.cipher.decryptor()
@@ -225,7 +231,7 @@ class CommunicationsHandler(QtCore.QObject):
                 messageJson = {"uuid": largeMessageUUID,
                                "message": partArg,
                                "done": done}
-                self.sock.send(self.encryptTransmission(dumps(messageJson)))
+                self.sock.send(self.encryptTransmission(dumps(messageJson)) + b'\x03\x03\x03\x03\x03')
         except BrokenPipeError:
             if showErrorOnBrokenPipe:
                 self.mainWindow.MESSAGEHANDLER.error("Disconnected from server!", popUp=True, exc_info=False)
@@ -270,14 +276,13 @@ class CommunicationsHandler(QtCore.QObject):
                     # Socket closed.
                     break
                 receivedInfo = oldData + receivedInfo
-                messages = receivedInfo.split(b'\x00\x00\x00')
-                if not receivedInfo.endswith(b'\x00\x00\x00'):
-                    oldData = messages[-1]
-                else:
-                    oldData = b''
+                messages = receivedInfo.split(b'\x03\x03\x03\x03\x03')
                 # Last message is either blank (i.e. '') or incomplete data, so we ignore it.
+                oldData = messages[-1]
                 messages = messages[:-1]
                 for message in messages:
+                    if len(message) == 0:
+                        continue
                     message = b64decode(message)
                     decryptedInfo = self.decryptTransmission(message)
                     if decryptedInfo is None:
@@ -309,6 +314,52 @@ class CommunicationsHandler(QtCore.QObject):
                 # They will be reconstructed eventually, once all the pieces get here.
                 continue
 
+    def askServerForCollectors(self, continuing_collectors_dict: dict) -> None:
+        message = {"Operation": "Get Server Collectors",
+                   "Arguments": {
+                       'continuing_collectors_dict': continuing_collectors_dict
+                   }}
+        self.transmitMessage(message)
+
+    def receiveStartCollector(self, collector_category: str, collector_name: str, collector_uid: str,
+                              collector_entities: list, collector_parameters: dict):
+        self.receive_start_collector_signal.emit(collector_category, collector_name, collector_uid,
+                                                 collector_entities, collector_parameters)
+
+    def receiveCollectors(self, server_collectors: dict, continuing_collectors_info: dict) -> None:
+        self.receive_collectors_signal.emit(server_collectors, continuing_collectors_info)
+
+    def startServerCollector(self, collector_name: str, collector_entities: list, collector_parameters: dict,
+                             continueTimestamp: int = 0) -> None:
+        collector_entities_to_send = []
+        for entity in collector_entities:
+            try:
+                dereferenced_entity = dict(entity)
+                collector_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any collector as of now: 2022/4/3.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+            except KeyError:
+                pass
+        message = {'Operation': 'Start Server Collector',
+                   'Arguments': {
+                       'collector_name': collector_name,
+                       'collector_entities': collector_entities_to_send,
+                       'collector_parameters': collector_parameters,
+                       'continue_time': continueTimestamp
+                   }}
+        self.transmitMessage(message)
+
+    def stopServerCollector(self, collector_uid: str) -> None:
+        message = {"Operation": "Stop Server Collector",
+                   "Arguments": {
+                       'collector_uid': collector_uid
+                   }}
+        self.transmitMessage(message)
+
+    def receiveCollectorResult(self, collector_name: str, collector_uid: str, timestamp: str, results: list) -> None:
+        self.receive_collector_result_signal.emit(collector_name, collector_uid, timestamp, results)
+
     def askServerForResolutions(self) -> None:
         message = {"Operation": "Get Server Resolutions",
                    "Arguments": {}}
@@ -589,6 +640,8 @@ class CommunicationsHandler(QtCore.QObject):
                 self.receiveProjectCanvasesList(**arguments)
             elif operation == 'Resolution Result':
                 self.receiveResolutionResult(**arguments)
+            elif operation == 'Collector Results Signal':
+                self.receiveCollectorResult(**arguments)
             elif operation == "Chat":
                 self.receiveTextMessage(**arguments)
             elif operation == "Sync Database":
@@ -609,6 +662,10 @@ class CommunicationsHandler(QtCore.QObject):
                 self.receiveFileSummaryListener(**arguments)
             elif operation == "File Upload Abort":
                 self.receiveFileUploadAbort(**arguments)
+            elif operation == "Get Server Collectors":
+                self.receiveCollectors(**arguments)
+            elif operation == "Start Collector":
+                self.receiveStartCollector(**arguments)
             else:
                 self.mainWindow.MESSAGEHANDLER.warning('Unhandled message: ' + str(message) +
                                                        ' On Operation: ' + str(operation))
@@ -677,6 +734,9 @@ class CommunicationsHandler(QtCore.QObject):
                 file_name = message.split(': ', 1)[1]
                 # Remove file from uploading files list.
                 self.file_upload_abort_signal.emit(file_name)
+            elif operation == 'Stop Collector':
+                # No need to do anything here - stopping collectors is only done by the client.
+                pass
             else:
                 self.mainWindow.MESSAGEHANDLER.warning('Unhandled status message: ' + message +
                                                        ' Code: ' + str(status_code) +

Core/GlobalVariables.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import random
+
+non_string_fields = ('Icon', 'Child UIDs')
+hidden_fields = ('uid', 'Date Last Edited', 'Child UIDs')
+meta_fields = ('Child UIDs',)
+avoid_parsing_fields = ('uid', 'Date Last Edited', 'Child UIDs', 'Icon')
+
+# Closer to the top means more recent.
+user_agents = {'Chrome': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                                      '(KHTML, like Gecko) Chrome/101.0.4951.15 Safari/537.36',),
+                          'Linux': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                                    'Chrome/101.0.4951.15 Safari/537.36',)
+                          },
+               'Firefox': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; rv:100.0) Gecko/20100101 Firefox/98.0',),
+                           'Linux': ('Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',)
+                           },
+               'Webkit': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 '
+                                      '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',),
+                          'Linux': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 '
+                                    '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',)
+                          }
+               }
+
+
+def getLatestUserAgent(browser: str = None, platform: str = None) -> str:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][0]
+
+
+def getUserAgents(browser: str = None, platform: str = None, amount: int = 1) -> tuple:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][:amount]

Core/Interface/CentralPane.py

@@ -4,7 +4,7 @@ import re
 import json
 import sys
 import threading
-from typing import Union
+from typing import Union, Optional
 from datetime import datetime
 
 import folium
@@ -18,7 +18,9 @@ from PySide6 import QtWidgets, QtGui, QtCore
 from PySide6.QtWidgets import QGraphicsPixmapItem
 from PySide6.QtSvgWidgets import QGraphicsSvgItem
 from PySide6.QtWebEngineWidgets import QWebEngineView
+
 from Core.Interface import Entity, Stylesheets
+from Core.ResourceHandler import RichNotesEditor
 
 
 class WorkspaceWidget(QtWidgets.QWidget):
@@ -28,7 +30,12 @@ class WorkspaceWidget(QtWidgets.QWidget):
                  messageHandler,
                  urlManager,
                  entityDB,
-                 resourceHandler):
+                 resourceHandler,
+                 entityTextFont,
+                 entityTextBrush,
+                 linkTextFont,
+                 linkTextBrush):
+
         super(WorkspaceWidget, self).__init__(parent=mainWindow)
 
         self.mainWindow = mainWindow
@@ -42,6 +49,10 @@ class WorkspaceWidget(QtWidgets.QWidget):
                                      urlManager,
                                      entityDB,
                                      resourceHandler,
+                                     entityTextFont,
+                                     entityTextBrush,
+                                     linkTextFont,
+                                     linkTextBrush,
                                      mainWindow)
 
         self.docAndCanvasLayout.addWidget(self.tabbedPane, 0, 1)
@@ -154,6 +165,10 @@ class TabbedPane(QtWidgets.QTabWidget):
                  urlManager,
                  entityDB,
                  resourceHandler,
+                 entityTextFont,
+                 entityTextBrush,
+                 linkTextFont,
+                 linkTextBrush,
                  mainWindow):
 
         super(TabbedPane, self).__init__(parent)
@@ -161,6 +176,10 @@ class TabbedPane(QtWidgets.QTabWidget):
         self.urlManager = urlManager
         self.entityDB = entityDB
         self.resourceHandler = resourceHandler
+        self.entityTextFont = entityTextFont
+        self.entityTextBrush = entityTextBrush
+        self.linkTextFont = linkTextFont
+        self.linkTextBrush = linkTextBrush
         self.mainWindow = mainWindow
 
         self.setAcceptDrops(True)
@@ -171,10 +190,18 @@ class TabbedPane(QtWidgets.QTabWidget):
         self.syncedTabs = []
         self.nodeCreationThreads = []
 
+        self.canvasDBPath = Path(self.mainWindow.SETTINGS.value("Project/BaseDir")) / "Project Files" / "CanvasTabs.lscanvas"
+        self.tabsNotesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("CanvasNotes.lsnotes")
+        self.tabsNotesDict = {}
+        self.previousTab = None
+
+        self.currentChanged.connect(self.currentTabChangedListener)
+
     def addCanvas(self, canvasName='New Graph', graph=None, positions=None, a=0, b=0, c=0, d=0) -> bool:
         if not self.isCanvasNameAvailable(canvasName):
             return False
-        scene = CanvasScene(self, graph, positions, a, b, c, d, canvasName)
+        scene = CanvasScene(self, graph, positions, a, b, c, d, canvasName,
+                            self.entityTextFont, self.entityTextBrush, self.linkTextFont, self.linkTextBrush)
         view = CanvasView(self,
                           scene,
                           canvasName,
@@ -232,19 +259,27 @@ class TabbedPane(QtWidgets.QTabWidget):
         return True
 
     def renameCanvas(self, currName: str, newName: str) -> None:
-        self.canvasTabs = {newName if key == currName else
-                           key: value for key, value in self.canvasTabs.items()}
+        self.canvasTabs[newName] = self.canvasTabs.pop(currName)
         self.canvasTabs[newName].name = newName
 
     def getViewAtIndex(self, index: int):
         return self.canvasTabs[self.tabText(index)]
 
     def closeTab(self, tabName: str) -> None:
+        """
+        Delete tab with name tabName.
+        @param tabName:
+        @return:
+        """
         for tabIndex in range(self.count()):
             if self.tabText(tabIndex) == tabName:
                 self.removeTab(tabIndex)
                 break
         self.canvasTabs.pop(tabName)
+        try:
+            self.tabsNotesDict.pop(tabName)
+        except KeyError:
+            pass
 
     def hideTab(self, index) -> None:
         self.removeTab(index)
@@ -332,6 +367,8 @@ class TabbedPane(QtWidgets.QTabWidget):
         allLinks = [linkUID['uid'] for linkUID in self.entityDB.getAllLinks()]
         links = []
         newNodeUIDs = []
+        nodesCreatedCount = 0
+        nodesUpdatedCount = 0
         for resultList in resolution_result:
             newNodeJSON = resultList[0]
             newNodeEntityType = newNodeJSON.get('Entity Type')
@@ -354,7 +391,10 @@ class TabbedPane(QtWidgets.QTabWidget):
                 del newNodeJSON[newNodePrimaryFieldKey]
                 try:
                     notesField = newNodeJSON.pop('Notes')
-                    existingEntityJSON['Notes'] += '\n' + notesField
+                    if existingEntityJSON.get('Notes'):
+                        existingEntityJSON['Notes'] += '\n' + notesField
+                    else:
+                        existingEntityJSON['Notes'] = str(notesField)
                 except KeyError:
                     # If no new field was actually added to the entity, don't re-add to the database
                     if len(newNodeJSON) == 0:
@@ -368,6 +408,7 @@ class TabbedPane(QtWidgets.QTabWidget):
                 existingEntityJSON.update(newNodeJSON)
                 self.entityDB.addEntity(existingEntityJSON, fromServer=True, updateTimeline=False)
                 newNodeUIDs.append(newNodeExistingUID)
+                nodesUpdatedCount += 1
             except ValueError:
                 # If there is no index for which the primary field and entity type of the new node match one of the
                 #   existing ones, the node must indeed be new. We add it here.
@@ -377,6 +418,7 @@ class TabbedPane(QtWidgets.QTabWidget):
                 #   create the same new entities.
                 allEntityUIDs.append(entityJson['uid'])
                 allEntityPrimaryFieldsAndTypes.append((newNodePrimaryField, newNodeEntityType))
+                nodesCreatedCount += 1
 
         progress.setValue(1)
         for resultListIndex in range(len(newNodeUIDs)):
@@ -387,6 +429,8 @@ class TabbedPane(QtWidgets.QTabWidget):
                 if isinstance(parentUID, int):
                     parentUID = newNodeUIDs[parentUID]
                 # Sanity check: Check that the node that was used for this resolution still exists.
+                #   If not, do not create link.
+                # Note: The new node was still created.
                 if parentUID in allEntityUIDs:
                     resolutionName = parentsDict[parentID].get('Resolution', 'Link')
                     newLinkUID = (parentUID, outputEntityUID)
@@ -411,7 +455,9 @@ class TabbedPane(QtWidgets.QTabWidget):
         progress.setValue(3)
         self.entityDB.resetTimeline()
         self.mainWindow.saveProject()
-        self.mainWindow.MESSAGEHANDLER.info('Resolution ' + resolution_name + ' completed successfully.')
+        self.mainWindow.MESSAGEHANDLER.info('Resolution ' + resolution_name + ' completed successfully: ' +
+                                            str(nodesCreatedCount) + ' new nodes created, ' + str(nodesUpdatedCount) +
+                                            ' existing nodes updated.')
 
     def linkAddHelper(self, links) -> None:
         """
@@ -437,7 +483,6 @@ class TabbedPane(QtWidgets.QTabWidget):
                        linkGroupingOverride: bool = False) -> None:
         for canvas in self.canvasTabs:
             scene = self.canvasTabs[canvas].scene()
-            sceneChanged = False
             with scene.resolutionThreadingLock:
                 addedNodes = []
 
@@ -445,28 +490,53 @@ class TabbedPane(QtWidgets.QTabWidget):
                     uid = newLink[1]
                     parentUID = newLink[0]
                     if parentUID in scene.nodesDict and uid not in scene.nodesDict:
-                        sceneChanged = True
+                        if uid not in scene.sceneGraph.nodes:
+                            nodeJSON = self.entityDB.getEntity(uid)
 
-                        nodeJSON = self.entityDB.getEntity(uid)
+                            # This is more efficient for large canvases than syncing afterwards.
+                            self.mainWindow.sendLocalCanvasUpdateToServer(canvas, uid)
 
-                        # This is more efficient for large canvases than syncing afterwards.
-                        self.mainWindow.sendLocalCanvasUpdateToServer(canvas, uid)
+                            picture = nodeJSON.get('Icon')
+                            scene.sceneGraph.add_node(uid)
 
-                        picture = nodeJSON.get('Icon')
-                        scene.sceneGraph.add_node(uid)
+                            try:
+                                nodePrimaryAttribute = nodeJSON.get(list(nodeJSON)[1])
+                            except IndexError:
+                                nodePrimaryAttribute = ''
+                            newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute, self.entityTextFont,
+                                                      self.entityTextBrush)
+                            scene.addNodeToScene(newNode)
 
-                        try:
-                            nodePrimaryAttribute = nodeJSON.get(list(nodeJSON)[1])
-                        except IndexError:
-                            nodePrimaryAttribute = ''
-                        newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
-                        scene.addNodeToScene(newNode)
-                        scene.addLinkDragDrop(scene.nodesDict[parentUID], newNode, newLink[2])
+                            scene.addLinkDragDrop(scene.nodesDict[parentUID], newNode, newLink[2])
+                            addedNodes.append(newNode)
+                        else:
+                            scene.addLinkProgrammatic((newLink[0], newLink[1]), newLink[2])
 
-                        addedNodes.append(newNode)
                     elif parentUID in scene.nodesDict and uid in scene.nodesDict:
                         # Need to send this to server, since it won't be drawn otherwise.
                         scene.addLinkDragDrop(scene.nodesDict[parentUID], scene.nodesDict[uid], newLink[2])
+                    elif parentUID not in scene.nodesDict and uid in scene.nodesDict:
+                        if parentUID not in scene.sceneGraph.nodes:
+                            nodeJSON = self.entityDB.getEntity(parentUID)
+
+                            # This is more efficient for large canvases than syncing afterwards.
+                            self.mainWindow.sendLocalCanvasUpdateToServer(canvas, parentUID)
+
+                            picture = nodeJSON.get('Icon')
+                            scene.sceneGraph.add_node(parentUID)
+
+                            try:
+                                nodePrimaryAttribute = nodeJSON.get(list(nodeJSON)[1])
+                            except IndexError:
+                                nodePrimaryAttribute = ''
+                            newNode = Entity.BaseNode(picture, parentUID, nodePrimaryAttribute, self.entityTextFont,
+                                                      self.entityTextBrush)
+                            scene.addNodeToScene(newNode)
+
+                            scene.addLinkDragDrop(newNode, scene.nodesDict[uid], newLink[2])
+                            addedNodes.append(newNode)
+                        else:
+                            scene.addLinkProgrammatic((newLink[0], newLink[1]), newLink[2])
 
                 if len(addedNodes) >= int(self.mainWindow.SETTINGS.value(
                         "Project/Resolution Result Grouping Threshold", "15")) and not linkGroupingOverride:
@@ -479,9 +549,9 @@ class TabbedPane(QtWidgets.QTabWidget):
                     [scene.removeNode(item) for item in addedNodes]
                     scene.addNodeProgrammatic(uid, itemUIDs, fromServer=True)
 
-            if sceneChanged:
-                scene.rearrangeGraph()
-                # self.mainWindow.syncCanvasByName(canvas)
+            # Even if nothing is added, the graph could need to be rearranged for clarity.
+            scene.rearrangeGraph()
+            # self.mainWindow.syncCanvasByName(canvas)
 
     def serverLinkAddHelper(self, linkJson: dict, overwrite: bool = False) -> None:
         """
@@ -517,35 +587,73 @@ class TabbedPane(QtWidgets.QTabWidget):
     def save(self) -> None:
         if len(self.canvasTabs) == 0:
             return
-        canvasDBPath = Path(self.mainWindow.SETTINGS.value("Project/BaseDir")) / "Project Files" / "canvasTabs.lscanvas"
-        canvasDBPathTmp = canvasDBPath.with_suffix(canvasDBPath.suffix + '.tmp')
 
-        canvasDBFile = open(canvasDBPathTmp, "wb")
-        saveJson = {}
-        for canvasName in self.canvasTabs:
-            saveJson[canvasName] = [
-                self.resourceHandler.deconstructGraphForFileDump(self.canvasTabs[canvasName].scene().sceneGraph),
-                self.canvasTabs[canvasName].scene().scenePos]
-        dump(saveJson, canvasDBFile)
-        canvasDBFile.close()
-        move(canvasDBPathTmp, canvasDBPath)
+        canvasDBPathTmp = self.canvasDBPath.with_suffix(self.canvasDBPath.suffix + '.tmp')
+        canvasNotesPathTmp = self.tabsNotesPath.with_suffix(self.tabsNotesPath.suffix + '.tmp')
+
+        # Save canvases
+        with open(canvasDBPathTmp, "wb") as canvasDBFile:
+            saveJson = {}
+            for canvasName in self.canvasTabs:
+                saveJson[canvasName] = [
+                    self.resourceHandler.deconstructGraphForFileDump(self.canvasTabs[canvasName].scene().sceneGraph),
+                    self.canvasTabs[canvasName].scene().scenePos]
+            dump(saveJson, canvasDBFile)
+        move(canvasDBPathTmp, self.canvasDBPath)
+
+        # Save canvas notes
+        currIndex = self.currentIndex()
+        if currIndex != -1:
+            canvasTabName = self.tabText(currIndex)
+            if canvasTabName in self.canvasTabs:
+                self.tabsNotesDict[canvasTabName] = self.mainWindow.dockbarTwo.getNotesText()
+        with open(canvasNotesPathTmp, "wb") as canvasNotesFileTmp:
+            dump(self.tabsNotesDict, canvasNotesFileTmp)
+        move(canvasNotesPathTmp, self.tabsNotesPath)
 
     def open(self) -> None:
-        canvasDBPath = Path(self.mainWindow.SETTINGS.value("Project/BaseDir")) / "Project Files" / "canvasTabs.lscanvas"
-
-        if Path(canvasDBPath).exists():
+        try:
+            with open(self.tabsNotesPath, 'rb') as notesFile:
+                # Load the content into the Notes pane.
+                self.tabsNotesDict = load(notesFile)
+        except ValueError:
+            # If the Tab Notes file is empty or contains invalid input, ignore it.
+            pass
+        except FileNotFoundError:
+            # Create new placeholder notes file if it doesn't exist.
             try:
-                canvasDBFile = open(canvasDBPath, "rb")
-                savedJson = load(canvasDBFile)
-                for canvasName in savedJson:
-                    self.addCanvas(canvasName,
-                                   self.resourceHandler.reconstructGraphFullFromFile(savedJson[canvasName][0]),
-                                   savedJson[canvasName][1])
-                canvasDBFile.close()
+                self.tabsNotesPath.touch(0o700, exist_ok=False)
+                self.mainWindow.MESSAGEHANDLER.info('Created new Tab Notes file.')
+            except FileExistsError:
+                self.mainWindow.MESSAGEHANDLER.error('Race condition occurred while trying to create '
+                                                     'Tab Notes file.')
+
+        if Path(self.canvasDBPath).exists():
+            try:
+                with open(self.canvasDBPath, "rb") as canvasDBFile:
+                    savedJson = load(canvasDBFile)
+                    for canvasName in savedJson:
+                        self.addCanvas(canvasName,
+                                       self.resourceHandler.reconstructGraphFullFromFile(savedJson[canvasName][0]),
+                                       savedJson[canvasName][1])
             except Exception as exc:
                 self.messageHandler.error("Exception occurred when opening tabs: " + str(exc) +
                                           "\nSkipping opening tabs.", popUp=True)
 
+    def currentTabChangedListener(self, newIndex: int) -> None:
+        if self.previousTab in self.canvasTabs:
+            # Will never have 'None' canvas, or consider deleted tabs.
+            self.tabsNotesDict[self.previousTab] = self.mainWindow.dockbarTwo.getNotesText()
+
+        if newIndex != -1:
+            canvasName = self.tabText(newIndex)
+            tabNotes = self.tabsNotesDict.get(canvasName, '#### Type notes here.\n')
+            self.mainWindow.dockbarTwo.setNotesText(tabNotes)
+            self.previousTab = canvasName
+        else:
+            self.mainWindow.dockbarTwo.setNotesText("")
+            self.previousTab = None
+
 
 class DocWorldPane(QtWidgets.QWidget):
 
@@ -581,8 +689,17 @@ class DocWorldPane(QtWidgets.QWidget):
                           if entity.get('Latitude') is not None and
                           entity.get('Longitude') is not None]
         for marker in markerEntities:
-            coordinates = [float(marker['Latitude'].replace(',', '.')),
-                           float(marker['Longitude'].replace(',', '.'))]
+            # Add check in case location attributes are already set as floats.
+            try:
+                latitude = float(marker['Latitude'].replace(',', '.'))
+            except AttributeError:
+                latitude = marker['Latitude']
+
+            try:
+                longitude = float(marker['Longitude'].replace(',', '.'))
+            except AttributeError:
+                longitude = marker['Longitude']
+            coordinates = [latitude, longitude]
             label = marker[list(marker)[1]]
             self.addMarker(coordinates, label)
         html = self.m.get_root().render()
@@ -641,11 +758,25 @@ class CanvasView(QtWidgets.QGraphicsView):
 
         self.menu = QtWidgets.QMenu()
         self.menu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
-        viewMenu = self.menu.addMenu("Hide / Delete")
+        selectMenu = self.menu.addMenu("Select...")
+        selectMenu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
+        viewMenu = self.menu.addMenu("Hide / Delete...")
         viewMenu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
-        groupingMenu = self.menu.addMenu("Grouping")
+        groupingMenu = self.menu.addMenu("Grouping...")
         groupingMenu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
 
+        actionSelectChildren = QtGui.QAction('Select Child Nodes',
+                                             selectMenu,
+                                             statusTip="Select the child entities of the selected nodes.",
+                                             triggered=self.scene().selectChildNodes)
+        selectMenu.addAction(actionSelectChildren)
+
+        actionSelectParents = QtGui.QAction('Select Parent Nodes',
+                                            selectMenu,
+                                            statusTip="Select the parent entities of the selected nodes.",
+                                            triggered=self.scene().selectParentNodes)
+        selectMenu.addAction(actionSelectParents)
+
         actionDelete = QtGui.QAction('Hide Selected Items',
                                      viewMenu,
                                      statusTip="Hide selected items from this canvas.",
@@ -746,11 +877,18 @@ class CanvasView(QtWidgets.QGraphicsView):
         if mimeData.hasUrls():
             jsonData = self.urlManager.handleURLs(mimeData.urls())
         elif mimeData.hasText():
+            mimeText = mimeData.text()
             try:
-                jsonData = [json.loads(mimeData.text())]
+                jsonData = [json.loads(mimeText)]
             except json.decoder.JSONDecodeError:
-                # Import as a Phrase entity
-                jsonData = [{'Phrase': mimeData.text(), 'Entity Type': 'Phrase'}]
+                # User has dragged in some text.
+                # Check if we should treat it as an url:
+                jsonData = self.urlManager.handleURLString(mimeText)
+                if jsonData is not None:
+                    jsonData = [jsonData]
+                elif jsonData:
+                    # If not, import as a Phrase entity, as long as the string is not blank.
+                    jsonData = [{'Phrase': mimeText, 'Entity Type': 'Phrase'}]
 
         if jsonData is None:
             return
@@ -877,7 +1015,7 @@ class CanvasView(QtWidgets.QGraphicsView):
                 self.zoom += 1
             else:
                 # Prevent user from zooming out indefinitely.
-                if self.zoom < -10:
+                if self.zoom < -11:
                     return
                 factor = 0.8
                 self.zoom -= 1
@@ -1038,7 +1176,8 @@ class CanvasView(QtWidgets.QGraphicsView):
 
 class CanvasScene(QtWidgets.QGraphicsScene):
 
-    def __init__(self, parent, graph=None, positions=None, a=0, b=0, c=0, d=0, canvasName: str = 'New Canvas') -> None:
+    def __init__(self, parent, graph=None, positions=None, a=0, b=0, c=0, d=0, canvasName: str = 'New Canvas',
+                 entityTextFont=None, entityTextBrush=None, linkTextFont=None, linkTextBrush=None) -> None:
         super(CanvasScene, self).__init__(a, b, c, d, parent)
         self.itemsToLink = []
         self.linking = False
@@ -1046,6 +1185,10 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         self.appendingToGroup = False
         self.sceneGraph = graph
         self.scenePos = positions
+        self.entityTextFont = entityTextFont
+        self.entityTextBrush = entityTextBrush
+        self.linkTextFont = linkTextFont
+        self.linkTextBrush = linkTextBrush
 
         # All the nodes on the canvas. Easier than looping through self.items().
         self.nodesDict = {}
@@ -1063,6 +1206,23 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         self.selectionChanged.connect(self.selectionChangeUpdater)
         self.resolutionThreadingLock = threading.Lock()
 
+    def updateNodeGraphics(self, entityTextFont, entityTextBrush, linkTextFont, linkTextBrush) -> None:
+        self.entityTextFont = entityTextFont
+        self.entityTextBrush = entityTextBrush
+        self.linkTextFont = linkTextFont
+        self.linkTextBrush = linkTextBrush
+        for item in self.items():
+            if isinstance(item, Entity.BaseNode):
+                item.labelItem.setFont(self.entityTextFont)
+                item.labelItem.setDefaultTextColor(self.entityTextBrush.color())
+                # Re-Center the Label
+                item.updateLabel(item.labelItem.toPlainText())
+            elif isinstance(item, Entity.BaseConnector):
+                item.labelItem.setFont(self.linkTextFont)
+                item.labelItem.setBrush(self.linkTextBrush)
+                # Re-Center the Label
+                item.updateLabel(item.labelItem.text())
+
     # Redefined so that the BaseConnector items are not considered.
     def itemsBoundingRect(self) -> QtCore.QRectF:
         try:
@@ -1073,10 +1233,10 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                     itemsX.append(item.pos().x())
                     itemsY.append(item.pos().y())
 
-            minX = min(itemsX) - 210
-            minY = min(itemsY) - 110
-            width = max(itemsX) - minX + 260
-            height = max(itemsY) - minY + 110
+            minX = min(itemsX) - (21 * self.entityTextFont.pointSize())
+            minY = min(itemsY) - 100 - self.entityTextFont.pointSize()
+            width = max(itemsX) - minX + (24 * self.entityTextFont.pointSize())
+            height = max(itemsY) - minY + 100 + self.entityTextFont.pointSize()
 
             return QtCore.QRectF(minX, minY, width, height)
         except ValueError:
@@ -1086,7 +1246,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         self.nodesDict[item.uid] = item
         self.addItem(item)
         item.setPos(QtCore.QPointF(x, y))
-        self.parent().mainWindow.MESSAGEHANDLER.info('Added node: ' + str(item.uid) + ' | ' + item.labelItem.text())
+        self.parent().mainWindow.MESSAGEHANDLER.info('Added node: ' + str(item.uid) + ' | ' +
+                                                     item.labelItem.toPlainText())
 
     def addLinkToScene(self, link: Entity.BaseConnector) -> None:
         self.linksDict[link.startItem().uid + link.endItem().uid] = link
@@ -1129,11 +1290,12 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 self.parent().mainWindow.toggleLinkingMode()
 
         elif self.appendingToGroup:
-            selectedGroupItems = [item for item in self.selectedItems()
-                                  if isinstance(item, Entity.GroupNode)]
+            selectedGroupItems = [item for item in self.selectedItems() if isinstance(item, Entity.GroupNode)]
             if len(selectedGroupItems) < 1:
                 pass
             elif len(selectedGroupItems) > 1:
+                self.parent().mainWindow.MESSAGEHANDLER.warning('Too many group items selected, '
+                                                                'please only choose one.', popUp=True)
                 self.parent().mainWindow.setStatus('Too many group items selected, please only choose one.')
             else:
                 groupEntityMaybe = selectedGroupItems[0]
@@ -1147,9 +1309,9 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                     newJson = self.parent().entityDB.getEntity(groupEntityMaybe.uid)
                     for item in self.itemsToAppendToGroup:
                         self.removeNode(item)
+                        self.sceneGraph.add_node(item.uid, groupID=groupEntityMaybe.uid)
                         groupEntityMaybe.addItemToGroup(item.uid)
                         newJson['Child UIDs'].append(item.uid)
-                        self.sceneGraph.add_node(item.uid, groupID=groupEntityMaybe.uid)
                     self.parent().entityDB.addEntity(newJson)
                     self.appendSelectedItemsToGroupToggle()
                     # Refresh list to show new entity/entities
@@ -1224,10 +1386,12 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 nodePrimaryAttribute = ''
 
             if groupItems is None:
-                newNode = Entity.BaseNode(picture, node, nodePrimaryAttribute)
+                newNode = Entity.BaseNode(picture, node, nodePrimaryAttribute, self.entityTextFont,
+                                          self.entityTextBrush)
                 self.addNodeToScene(newNode)
             else:
-                newNode = Entity.GroupNode(picture, node, nodePrimaryAttribute)
+                newNode = Entity.GroupNode(picture, node, nodePrimaryAttribute, self.entityTextFont,
+                                           self.entityTextBrush)
                 self.addNodeToScene(newNode)
 
                 newGroupList = newNode.listWidget
@@ -1246,6 +1410,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             self.addEntityLinkCreatorHelper(self.nodesDict[entity])
 
         progress.setValue(steps)
+        self.adjustSceneRect()
         self.parent().mainWindow.MESSAGEHANDLER.info('Loaded canvas: ' + canvasName)
 
     def updatePositionInDB(self, uid, x, y) -> None:
@@ -1276,7 +1441,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             #   either a new one being created or an old one that was copied.
             groupItems = [uid for uid in entity['Child UIDs'] if uid not in self.sceneGraph.nodes]
             if len(groupItems) > 0:
-                newNode = Entity.GroupNode(picture, uid, entity['Group Name'])
+                newNode = Entity.GroupNode(picture, uid, entity['Group Name'], self.entityTextFont,
+                                           self.entityTextBrush)
                 self.addNodeToScene(newNode, x, y)
 
                 newGroupList = newNode.listWidget
@@ -1288,7 +1454,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         elif entity.get('Entity Type'):
             if not fromServer:
                 self.parent().mainWindow.sendLocalCanvasUpdateToServer(self.getSelfName(), uid)
-            newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
+            newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute, self.entityTextFont,
+                                      self.entityTextBrush)
             self.addNodeToScene(newNode, x, y)
 
         if newNode is not None:
@@ -1319,12 +1486,14 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 nodePrimaryAttribute = entity.get(list(entity)[1])
             except IndexError:
                 nodePrimaryAttribute = ''
-            newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
+            newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute, self.entityTextFont,
+                                      self.entityTextBrush)
             self.addNodeToScene(newNode)
         else:
             groupItems = [uid for uid in groupItems if uid not in self.sceneGraph.nodes]
             if len(groupItems) > 0:
-                newNode = Entity.GroupNode(picture, uid, entity['Group Name'])
+                newNode = Entity.GroupNode(picture, uid, entity['Group Name'], self.entityTextFont,
+                                           self.entityTextBrush)
                 self.addNodeToScene(newNode)
 
                 newGroupList = newNode.listWidget
@@ -1342,72 +1511,73 @@ class CanvasScene(QtWidgets.QGraphicsScene):
 
         return newNode
 
-    def rearrangeGraph(self) -> None:
+    def rearrangeGraph(self, graphAlgorithm: str = None) -> None:
         # https://gitlab.com/graphviz/graphviz/-/merge_requests/2236
         # No triangulation library on windows yet, so sfdp can't be used there.
 
-        graphAlgorithm = self.parent().mainWindow.SETTINGS.value("Program/GraphLayout", 'dot')
+        if graphAlgorithm is None or graphAlgorithm not in ('sfdp', 'neato', 'dot', 'circular'):
+            graphAlgorithm = self.parent().mainWindow.SETTINGS.value("Program/GraphLayout", 'dot')
+
+        # No real 'links' to group nodes by default (links to internal nodes don't count). This means that the
+        #   graph algorithms can create odd graphs where group nodes are concerned.
+        # To fix this, we will duplicate the scene graph, add links between the group nodes and the
+        #   outside world, and use that clone for positions.
+        currGraphClone = nx.DiGraph(self.sceneGraph)
+
+        nodesToDel = set()
+        for edgeParentUID, edgeChild in dict(currGraphClone.edges):
+            potentialGroupIDOne = currGraphClone.nodes[edgeParentUID].get('groupID', edgeParentUID)
+            potentialGroupIDTwo = currGraphClone.nodes[edgeChild].get('groupID', edgeChild)
+            if potentialGroupIDOne != edgeParentUID:
+                nodesToDel.add(edgeParentUID)
+            if potentialGroupIDTwo != edgeChild:
+                nodesToDel.add(edgeChild)
+            # Nothing happens if edge already exists. If not, edge is created.
+            # These edges will not be visible on the actual graph generated.
+            currGraphClone.add_edge(potentialGroupIDOne, potentialGroupIDTwo)
+
+        # Only use nodes that are represented on the graph, else we can end up with nodes on the
+        #   other side of the world.
+        currGraphClone.remove_nodes_from(nodesToDel)
+
+        pdGraph = nx.nx_pydot.to_pydot(currGraphClone)
+
         try:
             if graphAlgorithm == 'sfdp':
-                self.scenePos = nx.nx_pydot.graphviz_layout(self.sceneGraph, 'sfdp')
+                pdGraph.set_layout('sfdp')
+                pdGraphNX = nx.nx_pydot.from_pydot(pdGraph)
+                self.scenePos = nx.nx_pydot.pydot_layout(pdGraphNX)
                 xFactor = (0.40 + min(len(self.nodesDict) / 100, 10))
                 yFactor = (0.40 + min(len(self.nodesDict) / 100, 10))
             elif graphAlgorithm == 'neato':
-                self.scenePos = nx.nx_pydot.graphviz_layout(self.sceneGraph, 'neato')
+                pdGraph.set_layout('neato')
+                pdGraphNX = nx.nx_pydot.from_pydot(pdGraph)
+                self.scenePos = nx.nx_pydot.pydot_layout(pdGraphNX)
                 xFactor = (3 + (0.30 + min(len(self.nodesDict) / 100, 10)))
                 yFactor = (3 + (0.30 + min(len(self.nodesDict) / 100, 10)))
+            elif graphAlgorithm == 'circular':
+                pdGraph.set_layout('circo')
+                pdGraphNX = nx.nx_pydot.from_pydot(pdGraph)
+                self.scenePos = nx.nx_pydot.pydot_layout(pdGraphNX)
+                xFactor = (0.40 + (0.30 + min(len(self.nodesDict) / 100, 10)))
+                yFactor = (0.40 + (0.30 + min(len(self.nodesDict) / 100, 10)))
             else:
                 # Default algorithm is 'dot', when nothing else is selected.
-
-                # No real 'links' to group nodes by default (links to internal nodes don't count). This means that the
-                #   'dot' algorithm can create odd graphs where group nodes are concerned.
-                # To fix this, we will duplicate the scene graph, add links between the group nodes and the
-                #   outside world, and use that clone for positions.
-                currGraphClone = nx.DiGraph(self.sceneGraph)
-
-                for edgeParentUID, edgeChild in dict(currGraphClone.edges):
-                    potentialGroupIDOne = currGraphClone.nodes[edgeParentUID].get('groupID', edgeParentUID)
-                    potentialGroupIDTwo = currGraphClone.nodes[edgeChild].get('groupID', edgeChild)
-                    # Nothing happens if edge already exists. If not, edge is created.
-                    # These edges will not be visible on the actual graph generated.
-                    currGraphClone.add_edge(potentialGroupIDOne, potentialGroupIDTwo)
-
-                # Only use nodes that are represented on the graph, else we can end up with nodes on the
-                #   other side of the world.
-                currGraphClone = nx.DiGraph(self.sceneGraph)
-                for node in dict(currGraphClone.nodes):
-                    if currGraphClone.nodes[node].get('groupID') is not None:
-                        currGraphClone.remove_node(node)
-
-                pdGraph = nx.drawing.nx_pydot.to_pydot(currGraphClone)
                 pdGraph.set_layout('dot')
                 pdGraph.set_rankdir('BT')
                 pdGraphNX = nx.nx_pydot.from_pydot(pdGraph)
-                self.scenePos = nx.drawing.nx_pydot.pydot_layout(pdGraphNX)
+                self.scenePos = nx.nx_pydot.pydot_layout(pdGraphNX)
                 xFactor = 0.70
                 yFactor = 2.75
         except Exception as exc:
             self.parent().mainWindow.MESSAGEHANDLER.error('Failed drawing graph with selected algorithm, falling back '
                                                           'to using "dot" algorithm: ' + str(exc), popUp=False)
             # If something goes wrong (e.g. the selected algorithm isn't found), use the dot algorithm.
-            # See comments for 'dot' algorithm.
-            currGraphClone = nx.DiGraph(self.sceneGraph)
 
-            for edgeParentUID, edgeChild in dict(currGraphClone.edges):
-                potentialGroupIDOne = currGraphClone.nodes[edgeParentUID].get('groupID', edgeParentUID)
-                potentialGroupIDTwo = currGraphClone.nodes[edgeChild].get('groupID', edgeChild)
-                currGraphClone.add_edge(potentialGroupIDOne, potentialGroupIDTwo)
-
-            currGraphClone = nx.DiGraph(self.sceneGraph)
-            for node in dict(currGraphClone.nodes):
-                if currGraphClone.nodes[node].get('groupID') is not None:
-                    currGraphClone.remove_node(node)
-
-            pdGraph = nx.drawing.nx_pydot.to_pydot(currGraphClone)
             pdGraph.set_layout('dot')
             pdGraph.set_rankdir('BT')
             pdGraphNX = nx.nx_pydot.from_pydot(pdGraph)
-            self.scenePos = nx.drawing.nx_pydot.pydot_layout(pdGraphNX)
+            self.scenePos = nx.nx_pydot.pydot_layout(pdGraphNX)
             xFactor = 0.70
             yFactor = 2.75
 
@@ -1518,18 +1688,20 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             linkToEdit.uid.add(linkUID)
         else:
             self.sceneGraph.add_edge(origin.uid, destination.uid)
-            self.addLinkToScene(Entity.BaseConnector(origin, destination, name))
+            self.addLinkToScene(Entity.BaseConnector(origin, destination, name, font=self.linkTextFont,
+                                                     brush=self.linkTextBrush))
 
         if not fromServer:
             self.parent().mainWindow.sendLocalCanvasUpdateToServer(self.getSelfName(), linkUID)
 
-    def addLinkProgrammatic(self, uid: tuple, name: str = 'None', fromServer: bool = False) -> None:
+    def addLinkProgrammatic(self, uid: tuple, name: str = 'None', fromServer: bool = False,
+                            suppressNonExistentEntityException: bool = False) -> None:
         if uid is None:
             self.parent().messageHandler.error("Cannot add link to canvas: Invalid link UID.")
             return
 
-        parentItem = self.getVisibleNodeForUID(uid[0])
-        childItem = self.getVisibleNodeForUID(uid[1])
+        parentItem = self.getVisibleNodeForUID(uid[0], suppressNonExistentEntityException)
+        childItem = self.getVisibleNodeForUID(uid[1], suppressNonExistentEntityException)
         if parentItem is None or childItem is None:
             return
 
@@ -1541,17 +1713,19 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             linkToEdit.uid.add(uid)
         else:
             self.sceneGraph.add_edge(uid[0], uid[1])
-            self.addLinkToScene(Entity.BaseConnector(parentItem, childItem, name, uid))
+            self.addLinkToScene(Entity.BaseConnector(parentItem, childItem, name, uid, font=self.linkTextFont,
+                                                     brush=self.linkTextBrush))
 
         if not fromServer:
             self.parent().mainWindow.sendLocalCanvasUpdateToServer(self.getSelfName(), uid)
 
-    def getVisibleNodeForUID(self, uid: str):
+    def getVisibleNodeForUID(self, uid: str, suppressNonExistentEntityException: bool = False):
         """
         This gets the node on the canvas that the uid corresponds to, or the visible group node that the entity
         with the given uid is in.
 
         :param uid:
+        :param suppressNonExistentEntityException: Suppresses errors where entities that should exist, do not (yet).
         :return:
         """
         # There is an assumption made here: That a canvas that contains a grouped node will also contain the group
@@ -1566,8 +1740,9 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             try:
                 return self.nodesDict[uid]
             except KeyError:
-                self.parent().messageHandler.error('Canvas state is undefined: Tried to get a node not present in the '
-                                                   'canvas, uid: ' + str(uid))
+                if not suppressNonExistentEntityException:
+                    self.parent().messageHandler.error('Canvas state may be undefined: Tried to get a node not '
+                                                       'present in the canvas, with uid: ' + str(uid))
         return None
 
     def getVisibleLinkForUID(self, uid: tuple):
@@ -1592,8 +1767,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
 
         for edgeToDelete in edgesToDelete:
             # Remove UIDs from list, and delete the link if no more UIDs are left
-            self.linksDict[edgeToDelete].uid = [linkToStayUID for linkToStayUID in self.linksDict[edgeToDelete].uid
-                                                if linkToStayUID not in edgesToDelete[edgeToDelete]]
+            self.linksDict[edgeToDelete].uid = {linkToStayUID for linkToStayUID in self.linksDict[edgeToDelete].uid
+                                                if linkToStayUID not in edgesToDelete[edgeToDelete]}
             if len(self.linksDict[edgeToDelete].uid) == 0:
                 self.removeEdge(self.linksDict[edgeToDelete])
 
@@ -1653,9 +1828,38 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             self.parent().messageHandler.info('Edited link: ' + str(pEditor.objectJson['uid']) + ' | ' +
                                               pEditor.objectJson['Resolution'])
 
+    def selectAllNodes(self) -> None:
+        self.clearSelection()
+        for node in self.nodesDict:
+            self.nodesDict[node].setSelected(True)
+
+    def selectChildNodes(self) -> None:
+        items = [item.uid for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
+        self.clearSelection()
+        for item in items:
+            childLinks = self.parent().entityDB.getOutgoingLinks(item)
+            for childLink in childLinks:
+                # Ask forgiveness instead of permission - set children as selected, if they are on the canvas.
+                try:
+                    self.nodesDict[childLink[1]].setSelected(True)
+                except KeyError:
+                    pass
+
+    def selectParentNodes(self) -> None:
+        items = [item.uid for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
+        self.clearSelection()
+        for item in items:
+            parentLinks = self.parent().entityDB.getIncomingLinks(item)
+            for parentLink in parentLinks:
+                # Ask forgiveness instead of permission - set parents as selected, if they are on the canvas.
+                try:
+                    self.nodesDict[parentLink[0]].setSelected(True)
+                except KeyError:
+                    pass
+
     # Because the entities on each canvas are stored in dicts, and dicts are ordered, group nodes will always
     # come after the nodes they contain.
-    def groupSelectedItems(self):
+    def groupSelectedItems(self) -> Optional[Entity.GroupNode]:
         items = [item for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
         for item in items:
             if isinstance(item, Entity.GroupNode):
@@ -1676,6 +1880,16 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         else:
             self.parent().mainWindow.setStatus('Please select more than one node to create a Group node.')
 
+    def groupItemsProgrammatic(self, itemUIDs: set, groupName: str = 'Entity Group') -> Optional[Entity.GroupNode]:
+        newGroupUIDs = [uid for uid in itemUIDs if not uid.endswith('@')]
+        if len(newGroupUIDs) > 1:
+            newEntity = self.parent().entityDB.addEntity(
+                {'Group Name': groupName, 'Child UIDs': newGroupUIDs, 'Entity Type': 'EntityGroup'})
+            uid = newEntity['uid']
+            [self.removeNode(self.nodesDict[uid]) for uid in newGroupUIDs]
+            return self.addNodeProgrammatic(uid, itemUIDs)
+        return None
+
     def ungroupSelectedItems(self) -> None:
         groups = [item for item in self.selectedItems() if isinstance(item, Entity.GroupNode)]
         for group in groups:
@@ -1791,7 +2005,7 @@ class PropertiesEditor(QtWidgets.QDialog):
                 continue
             keyField = QtWidgets.QLabel(key)
             if key == "Notes":
-                valueField = QtWidgets.QPlainTextEdit(objectJson[key])
+                valueField = RichNotesEditor(self, objectJson["Notes"])
             elif key == 'Icon':
                 valueField = PropertiesEditorIconField(objectJson[key], objectJson['uid'], self.canvas)
             elif key == 'File Path':
@@ -1817,7 +2031,7 @@ class PropertiesEditor(QtWidgets.QDialog):
                 row, self.itemProperties.LabelRole).widget().text()
             if key == "Notes":
                 value = self.itemProperties.itemAt(
-                    row, self.itemProperties.FieldRole).widget().toPlainText()
+                    row, self.itemProperties.FieldRole).widget().toMarkdown()
             elif key == 'Icon':
                 value = self.itemProperties.itemAt(
                     row, self.itemProperties.FieldRole).widget().pictureByteArray

Core/Interface/DockBarThree.py

@@ -50,6 +50,7 @@ class DockBarThree(QtWidgets.QDockWidget):
         self.chatBox = ChatBox(self, self.parent())
         self.timeWidget = TimeWidget(self, self.parent())
         self.logViewer = QtWidgets.QPlainTextEdit()
+        self.logViewer.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
 
         # Because we're not going to stop the thread before closing, an error will be thrown by Qt.
         # That error can be safely ignored.

Core/Interface/DockBarTwo.py

@@ -4,6 +4,7 @@ from pathlib import Path
 import magic
 from PySide6 import QtWidgets, QtCore, QtGui
 from Core.Interface import Stylesheets
+from Core.ResourceHandler import MinSizeStackedLayout, RichNotesEditor
 
 
 class DockBarTwo(QtWidgets.QDockWidget):
@@ -15,12 +16,12 @@ class DockBarTwo(QtWidgets.QDockWidget):
 
         scrollAreaWidget.setWidget(self.entDetails)
         scrollAreaWidget.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
-        scrollAreaWidget.setMinimumWidth(485)
+        scrollAreaWidget.setWidgetResizable(True)
+        scrollAreaWidget.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum)
 
         childWidget.addTab(scrollAreaWidget, 'Entity Details')
         childWidget.addTab(self.oracle, 'Oracle')
-        self.setMaximumWidth(600)
-        self.resize(self.height(), 500)
+        childWidget.addTab(self.tabNotes, 'Notes')
 
     def __init__(self,
                  mainWindow,
@@ -44,9 +45,42 @@ class DockBarTwo(QtWidgets.QDockWidget):
                                         self.entityDB,
                                         self.parent(),
                                         self)
+        self.tabNotes = TabNotesPanel(self)
 
         self.initialiseLayout()
 
+    def setNotesText(self, newText: str) -> None:
+        if self.tabNotes.textEditor.isReadOnly():
+            self.tabNotes.textEditor.contents = newText
+            self.tabNotes.textEditor.setMarkdown(newText)
+        else:
+            self.tabNotes.textEditor.setReadOnly(True)
+            self.tabNotes.textEditor.contents = newText
+            self.tabNotes.textEditor.setMarkdown(newText)
+
+    def getNotesText(self) -> str:
+        """
+        Get text with the Markdown control characters.
+        @return:
+        """
+        return self.tabNotes.textEditor.toMarkdown()
+
+
+class TabNotesPanel(QtWidgets.QWidget):
+    """
+    This class is used to allow the user to take notes on a tab by tab
+    basis.
+    """
+
+    def __init__(self, parent=None):
+        super(TabNotesPanel, self).__init__(parent=parent)
+        tabNotesLayout = QtWidgets.QVBoxLayout()
+
+        self.textEditor = RichNotesEditor(self)
+
+        tabNotesLayout.addWidget(self.textEditor)
+        self.setLayout(tabNotesLayout)
+
 
 class EntityDetails(QtWidgets.QWidget):
     """
@@ -65,8 +99,9 @@ class EntityDetails(QtWidgets.QWidget):
         self.mainWindow = mainWindow
         self.resourceHandler = resourceHandler
         self.entityDB = entityDB
-        self.detailsLayout = QtWidgets.QStackedLayout()
+        self.detailsLayout = MinSizeStackedLayout()
         self.setLayout(self.detailsLayout)
+        self.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum)
 
         layoutNothingSelected = QtWidgets.QVBoxLayout()
         widgetNothing = QtWidgets.QWidget()
@@ -86,8 +121,7 @@ class EntityDetails(QtWidgets.QWidget):
         # Need to keep track of how many nodes are selected.
         # ~ Nothing Selected/Hovered Layout
         nothingLabel = QtWidgets.QLabel("Nothing is Selected.")
-        nothingLabel.setAlignment(QtCore.Qt.AlignHCenter |
-                                  QtCore.Qt.AlignVCenter)
+        nothingLabel.setAlignment(QtCore.Qt.AlignCenter)
         layoutNothingSelected.addWidget(nothingLabel)
         ###
 
@@ -97,6 +131,7 @@ class EntityDetails(QtWidgets.QWidget):
         summaryPanel = QtWidgets.QWidget()
         summaryPanel.setLayout(summaryLayout)
         self.summaryIcon = QtWidgets.QLabel("")
+        self.summaryIcon.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum)
         self.entityTypeLabel = QtWidgets.QLabel("")
         self.entityUIDLabel = QtWidgets.QLabel("")
         self.entityPrimaryLabel = QtWidgets.QLabel("")
@@ -171,8 +206,6 @@ class EntityDetails(QtWidgets.QWidget):
         layoutMultipleItemsSelected.addWidget(self.multiNodesTableLabelTwo)
         layoutMultipleItemsSelected.addWidget(self.linksTable)
 
-        self.setFixedWidth(475)
-
         self.currentlyShown = []
 
     def displayWidgetDetails(self, jsonDicts) -> None:
@@ -249,8 +282,7 @@ class EntityDetails(QtWidgets.QWidget):
             if key == "uid" or key == "Child UIDs" or key == "Icon":
                 continue
             elif key == "Notes":
-                notesTextArea = QtWidgets.QPlainTextEdit(jsonDict[key])
-                notesTextArea.setReadOnly(True)
+                notesTextArea = RichNotesEditor(self, jsonDict[key], False)
                 self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
                 self.detailsLayoutOneNode.addWidget(notesTextArea, rowCount, 1, 10, 1)
                 rowCount += 9
@@ -385,10 +417,10 @@ class SingleLinkItem(QtWidgets.QWidget):
         super().__init__(parent=parent)
 
         self.linkItemPic = QtWidgets.QLabel()
-        
+
         self.linkItemPic.setAlignment(QtCore.Qt.AlignCenter)
         self.linkItemName = QtWidgets.QLabel()
-        
+
         self.linkItemName.setAlignment(QtCore.Qt.AlignCenter)
         self.linkItemUid = ""
         self.setMaximumHeight(90)

Core/Interface/Entity.py

@@ -6,15 +6,17 @@ from typing import Any, Optional
 
 from PySide6 import QtCore, QtGui, QtWidgets
 from PySide6.QtWidgets import QGraphicsItem
-from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem
+from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem, QGraphicsTextItem
 from PySide6.QtSvgWidgets import QGraphicsSvgItem
 
-TEXTFONT = QtGui.QFont("Mono", 11, 700)
+ENTITY_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
+LINK_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
 
 
 class BaseNode(QGraphicsItemGroup):
 
-    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str) -> None:
+    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str, font: QtGui.QFont,
+                 brush: QtGui.QBrush) -> None:
         super(BaseNode, self).__init__()
 
         self.setCacheMode(self.DeviceCoordinateCache)
@@ -31,11 +33,26 @@ class BaseNode(QGraphicsItemGroup):
         else:
             self.iconItem = QGraphicsPixmapItem(self.pixmapItem)
 
-        self.labelItem = QGraphicsSimpleTextItem('')
+        self.labelItem = QGraphicsTextItem('')
+        # Have to do it this way; directly assigning stuff does not work due to how PySide6 works.
+        labelDocument = self.labelItem.document()
+        labelDocument.setTextWidth(280)
+        textOption = labelDocument.defaultTextOption()
+        textOption.setWrapMode(QtGui.QTextOption.WrapAtWordBoundaryOrAnywhere)
+        textOption.setAlignment(QtCore.Qt.AlignHCenter)
+        labelDocument.setDefaultTextOption(textOption)
+        self.labelItem.setDocument(labelDocument)
+
         self.addToGroup(self.iconItem)
         self.addToGroup(self.labelItem)
-        self.labelItem.setFont(TEXTFONT)
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(ENTITY_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setDefaultTextColor(brush.color())
 
+        self.labelItem.setPos(self.iconItem.x() - 120, self.iconItem.y() + 45)
         self.updateLabel(primaryAttribute)
 
         self.uid = uid
@@ -51,12 +68,12 @@ class BaseNode(QGraphicsItemGroup):
         self.parentGroup = None
 
     def updateLabel(self, newText: str = '') -> None:
+        if not isinstance(newText, str):
+            newText = str(newText)
         if newText != '':
             if len(newText) > 50:
                 newText = newText[:47] + "..."
-            self.labelItem.setText(newText)
-            labelWidth = self.labelItem.boundingRect().width()
-            self.labelItem.setPos(self.iconItem.x() + 20 - (labelWidth / 2), self.iconItem.y() + 45)
+            self.labelItem.setPlainText(newText)
 
     def removeConnector(self, connector) -> None:
         # Exception could be thrown if the connector is already deleted.
@@ -115,8 +132,8 @@ class BaseNode(QGraphicsItemGroup):
 class GroupNode(BaseNode):
 
     # childNodes is a list of tuples, uid and picture, of all the nodes in the group.
-    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group') -> None:
-        super(GroupNode, self).__init__(pictureByteArray, uid, label)
+    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group', font=None, brush=None) -> None:
+        super(GroupNode, self).__init__(pictureByteArray, uid, label, font, brush)
         self.groupedNodesConnectors = []
         self.itemsThatWereGrouped = []
         self.groupedNodesUid = set()
@@ -175,8 +192,13 @@ class GroupNode(BaseNode):
         incomingLinks = self.scene().parent().entityDB.getIncomingLinks(uid)
         outgoingLinks = self.scene().parent().entityDB.getOutgoingLinks(uid)
 
-        [self.scene().addLinkProgrammatic(link[0], self.uid) for link in incomingLinks]
-        [self.scene().addLinkProgrammatic(self.uid, link[1]) for link in outgoingLinks]
+        # Suppress exceptions, as these give false alarms when initially drawing the graph on the canvas.
+        for link in incomingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
+        for link in outgoingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
 
     def removeSpecificItemFromGroupIfExists(self, uid) -> bool:
         if uid in self.groupedNodesUid:
@@ -190,7 +212,8 @@ class GroupNode(BaseNode):
 # Ref: https://github.com/PySide/Examples/blob/master/examples/graphicsview/diagramscene/diagramscene.py
 class BaseConnector(QGraphicsItemGroup):
 
-    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None) -> None:
+    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None,
+                 font: QtGui.QFont = None, brush: QtGui.QBrush = None) -> None:
         super(BaseConnector, self).__init__(parent)
 
         self.myStartItem = origin
@@ -198,7 +221,12 @@ class BaseConnector(QGraphicsItemGroup):
 
         self.labelItem = QGraphicsSimpleTextItem('')
         self.addToGroup(self.labelItem)
-        self.labelItem.setFont(TEXTFONT)
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(LINK_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setBrush(brush)
 
         self.updateLabel(name)
 
@@ -369,6 +397,7 @@ class ChildListWidget(QtWidgets.QListWidget):
     def __init__(self) -> None:
         super(ChildListWidget, self).__init__()
         self.setAlternatingRowColors(True)
+        self.setSortingEnabled(True)
 
     def mouseMoveEvent(self, event: QtGui.QMouseEvent) -> None:
         itemDragged = self.itemAt(event.pos())

Core/Interface/Stylesheets.py

@@ -16,80 +16,120 @@ TOOLBAR_STYLESHEET = """QToolBar {background-color: rgb(41, 45, 62);
 
 MAIN_WINDOW_STYLESHEET = """
                         QWidget{ 
-                        background-color: rgb(41, 45, 62);
-                        color: rgba(248, 248, 242, 1) !important;
-                        font-size: 15px;
+                         background-color: rgb(41, 45, 62);
+                         color: rgba(248, 248, 242, 1) !important;
+                         font-size: 15px;
                         }
+
                         QScrollBar:vertical {         
-                        background:rgb(44, 49, 58);
-                        width:7px;    
-                        margin: 0px 0px 0px 0px;
+                         background:rgb(44, 49, 58);
+                         width:7px;    
+                         margin: 0px 0px 0px 0px;
                         }
+
                         QScrollBar::handle:vertical {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
-                            min-height: 0px;
+                         background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                         stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                         min-height: 0px;
                         }
+
                         QScrollBar:horizontal {         
-                            background:rgb(44, 49, 58);
-                            height:7px;    
-                            margin: 0px 0px 0px 0px;
+                         background:rgb(44, 49, 58);
+                         height:7px;    
+                         margin: 0px 0px 0px 0px;
                         }
+
                         QScrollBar::handle:horizontal {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                         background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                                                     stop: 0 rgb(103, 110, 149),
+                                                     stop: 0.5 rgb(103, 110, 149),
+                                                     stop:1 rgb(103, 110, 149));
                         }
                         QMenuBar { 
-                          color: #ffffff;
-                          background-color: rgb(33, 37, 43);
-                          font-size: 15px;
+                         color: #ffffff;
+                         background-color: rgb(33, 37, 43);
+                         font-size: 15px;
                         }
 
                         QPushButton {
-                        border: 2px solid rgb(0, 173, 238);
-                        border-radius: 6px;
-                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
-                        min-width: 50px;
+                         border: 2px solid rgb(0, 173, 238);
+                         border-radius: 6px;
+                         background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                           stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
+                         min-width: 50px;
                         }
 
                         QLabel { 
-                        border: 2px solid rgb(41, 45, 62);
-                        padding-left: 7px;
-                        border-left-color: rgb(0, 173, 238);
+                         border: 2px solid rgb(41, 45, 62);
+                         padding-left: 7px;
+                         border-left-color: rgb(0, 173, 238);
                         }
                         
                         QLineEdit, QPlainTextEdit { 
-                        border: 0.5px solid rgb(0, 173, 238);
-                        border-radius: 6px;
+                         border: 0.5px solid rgb(0, 173, 238);
+                         border-radius: 6px;
                         }
                         
-                        QTabBar::tab {background: rgb(68, 66, 103);
-                        border: 2px solid rgb(41, 45, 62);
-                        border-radius: 3px;
-                        min-height: 3ex;
+                        QTabBar::tab {
+                         background: rgb(68, 66, 103);
+                         border: 2px solid rgb(41, 45, 62);
+                         border-radius: 3px;
+                         min-height: 3ex;
                         }
+
                         QTabWidget {
-                        border-style: outset;
-                        border-color: rgba(248, 248, 242, 1);
-                        border-width: 0.5px;
+                         border-style: outset;
+                         border-color: rgba(248, 248, 242, 1);
+                         border-width: 0.5px;
                         }
+
                         QToolBar {
-                        border-style: outset;
-                        border-color: rgba(75, 75, 75, 1);
-                        border-width: 1px;
-                        border-left-width: 0px;
-                        border-right-width: 0px;
+                         border-style: outset;
+                         border-color: rgba(75, 75, 75, 1);
+                         border-width: 1px;
+                         border-left-width: 0px;
+                         border-right-width: 0px;
                         }
                         
-                        QTabBar::tab:selected {background: rgb(51, 55, 95);
-                        border: 2px solid rgb(41, 45, 62);
-                        min-height: 2.5ex;
-                        border-radius: 3px;
-                        border-top-color: rgb(0, 173, 238);
+                        QTabBar::tab:selected {
+                         background: rgb(51, 55, 95);
+                         border: 2px solid rgb(41, 45, 62);
+                         min-height: 2.5ex;
+                         border-radius: 3px;
+                         border-top-color: rgb(0, 173, 238);
                         }
+                        
                         QComboBox { combobox-popup: 0; }
                         
+                        QHeaderView::section { 
+                         background-color: rgb(41, 45, 62);
+                         color: rgba(248, 248, 242, 1);
+                        }
+                        
+                        QMenu::item {
+                         background-color: rgb(41, 45, 62);
+                         color: rgba(248, 248, 242, 1) !important;
+                         border-left: 1px solid rgb(0, 173, 238);
+                         padding-right: 4px;
+                         padding-bottom: 4px;
+                         padding-top: 4px;
+                         font-size: 15px;
+                         text-align: left;
+                         padding-left: 4px;
+                        }
+                        
+                        QMenu::item:selected {
+                         background-color: rgb(0, 85, 127);
+                         color: rgb(255, 255, 255);
+                        }
+                         
+                        QMenu::item:disabled {
+                         background-color:rgb(81, 87, 114);
+                        }
+                        
+                        QTextBrowser {
+                         background-color:rgb(60, 60, 80);
+                        }
                         """
 
 DOCK_BAR_TWO_LINK = """
@@ -108,13 +148,6 @@ DOCK_BAR_LABEL = """
                         }
                   """
 
-DROPDOWN_MENU_STYLE = "background-color: rgb(41, 45, 62);" \
-                      "color: rgba(248, 248, 242, 1) !important;" \
-                      "border: 2px solid rgb(44, 49, 58);" \
-                      "border-left-color: rgb(189, 147, 249);" \
-                      "font-family: Segoe UI;" \
-                      "font-size: 13px;"
-
 TEXT_BOX_STYLESHEET = "QLineEdit, QLabel{" \
                       "border-bottom: 1px solid rgb(0, 173, 238);" \
                       "}"

Core/Interface/ToolBarOne.py

@@ -18,6 +18,7 @@ class ToolBarOne(QtWidgets.QToolBar):
                                   statusTip="Create new Canvas or Open existing Canvas",
                                   triggered=self.addCanvas,
                                   icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('addCanvas')))
+        newCanvas.setShortcut('Ctrl+N')
         self.addAction(newCanvas)
         self.insertSeparator(newCanvas)
 
@@ -92,13 +93,13 @@ class ToolBarOne(QtWidgets.QToolBar):
         self.addAction(splitEntity)
         self.insertSeparator(splitEntity)
 
-        generateReport = QtGui.QAction('Generate Report',
-                                       self,
-                                       statusTip="Generate Report of selected entities.",
-                                       triggered=self.generateReports,
-                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
-        self.addAction(generateReport)
-        self.insertSeparator(generateReport)
+        generateReportAction = QtGui.QAction('Generate Report',
+                                             self,
+                                             statusTip="Generate Report from the selected nodes.",
+                                             triggered=self.generateReport,
+                                             icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
+        self.addAction(generateReportAction)
+        self.insertSeparator(generateReportAction)
 
         rearrangeCanvas = QtGui.QAction('Rearrange Canvas',
                                         self,
@@ -147,5 +148,5 @@ class ToolBarOne(QtWidgets.QToolBar):
     def splitEntity(self):
         self.parent().splitEntity()
 
-    def generateReports(self):
+    def generateReport(self):
         self.parent().generateReport()

Core/LQL.py

@@ -0,0 +1,502 @@
+#!/usr/bin/env python3
+
+from typing import Union, Optional, Any
+from uuid import uuid4
+import re
+import networkx as nx
+import string
+
+from Core.GlobalVariables import non_string_fields
+
+"""
+This class handles the backend stuff for the LinkScope Query Language.
+"""
+
+
+class LQLQueryBuilder:
+    QUERIES_HISTORY = {}
+
+    databaseSnapshot = None
+    databaseEntities = None
+    allCanvases = None
+    canvasesEntitiesDict = None
+    allEntityFields = None
+    allEntities = None
+
+    def __init__(self, mainWindow):
+        self.mainWindow = mainWindow
+
+    def takeSnapshot(self):
+        self.mainWindow.LENTDB.dbLock.acquire()
+        # Create a copy
+        self.databaseSnapshot = nx.DiGraph(self.mainWindow.LENTDB.database)
+        self.mainWindow.LENTDB.dbLock.release()
+
+        self.databaseEntities = set(self.databaseSnapshot.nodes)
+
+        self.allCanvases = self.getAllCanvasNames()
+        self.canvasesEntitiesDict = self.getCanvasesEntitiesDict(self.allCanvases)
+        self.allEntityFields, self.allEntities = self.getAllEntitiesAndFields()
+
+        # Re-define database entities to remove Group Entities
+        self.databaseEntities = set(self.allEntities.keys())
+
+    def getAllEntitiesAndFields(self) -> (set, list):
+        entitiesSnapshot = {entity: self.databaseSnapshot.nodes[entity] for entity in self.databaseSnapshot.nodes
+                            if self.databaseSnapshot.nodes[entity].get('Entity Type') != 'EntityGroup'}
+        entityFields = set()
+        for entityUID in entitiesSnapshot:
+            entityFields.update(entitiesSnapshot[entityUID].keys())
+        for field in non_string_fields:
+            try:
+                entityFields.remove(field)
+            except KeyError:
+                # This typically only happens if there are no entities in the database.
+                continue
+        return entityFields, entitiesSnapshot
+
+    def getAllCanvasNames(self) -> list:
+        canvasNames = list(self.mainWindow.centralWidget().tabbedPane.canvasTabs.keys())
+        canvasNames.append('*')
+        return canvasNames
+
+    def getEntitiesOnCanvas(self, canvasName: str):
+        try:
+            # Ensure that we don't have nodes here that are not present in our database snapshot
+            canvasNodes = set(
+                self.mainWindow.centralWidget().tabbedPane.canvasTabs[canvasName].scene().sceneGraph.nodes)
+            return canvasNodes.intersection(self.databaseEntities)
+        except KeyError:
+            return None
+
+    def getCanvasesEntitiesDict(self, allCanvasNames: list):
+        returnDict = {}
+        for canvas in allCanvasNames:
+            allEntitiesOnCanvas = self.getEntitiesOnCanvas(canvas)
+            if allEntitiesOnCanvas is not None:
+                returnDict[canvas] = allEntitiesOnCanvas
+            else:
+                returnDict[canvas] = set()
+
+        return returnDict
+
+    def parseSelect(self, selectClause: str, selectValue: Union[str, list]):
+        if selectClause == 'SELECT':
+            if '*' in selectValue:
+                # No need to remove the '*'. Could cause errors if that's a field name (even though it is bad practice).
+                return self.allEntityFields
+            return set([entityField for entityField in selectValue if entityField in self.allEntityFields])
+        else:
+            try:
+                clauseValue = re.compile(selectValue)
+                return set([entityField for entityField in self.allEntityFields if clauseValue.match(entityField)])
+            except re.error:
+                return set()
+
+    def parseSource(self, sourceClause: str, sourceValues: Union[None, list], fieldsToSelect: set) -> set:
+        """
+        sourceValues:
+        [[("AND" | "OR" | None), ("CANVAS" | "RCANVAS"), (True | False), <User Input>], ...]
+        OR
+        None
+            if sourceClause == "FROMDB"
+        """
+        if sourceClause == "FROMDB":
+            resultEntitySet = set(self.databaseEntities)
+        else:
+            resultEntitySet = set()
+            for sourceValue in sourceValues:
+                try:
+                    if sourceValue[1] == "CANVAS":
+                        if sourceValue[3] not in self.allCanvases:
+                            raise ValueError('Reference to nonexistent canvas.')
+                        matchingCanvases = [sourceValue[3]]
+                    else:
+                        canvasRegex = re.compile(sourceValue[3])
+                        matchingCanvases = [canvasMatch for canvasMatch in self.allCanvases
+                                            if canvasRegex.match(canvasMatch)]
+                except (ValueError, re.error):
+                    continue
+
+                # Not the most efficient way of phrasing this, but by far the most compact and legible.
+                for matchingCanvas in matchingCanvases:
+                    if sourceValue[0] == 'AND':
+                        if sourceValue[2] is True:
+                            resultEntitySet = self.canvasAndNot(resultEntitySet,
+                                                                self.canvasesEntitiesDict[matchingCanvas])
+                        else:
+                            resultEntitySet = self.canvasAnd(resultEntitySet,
+                                                             self.canvasesEntitiesDict[matchingCanvas])
+                    else:
+                        # If this is the first clause, or'ing the empty initial resultEntitySet is what we want.
+                        if sourceValue[2] is True:
+                            resultEntitySet = self.canvasOrNot(resultEntitySet,
+                                                               self.canvasesEntitiesDict[matchingCanvas],
+                                                               self.databaseEntities)
+                        else:
+                            resultEntitySet = self.canvasOr(resultEntitySet,
+                                                            self.canvasesEntitiesDict[matchingCanvas])
+
+        # Filter out all entities that do not contain at least one of the selected fields.
+        for entity in list(resultEntitySet):
+            validEntity = False
+            for field in fieldsToSelect:
+                if field in self.allEntities[entity].keys():
+                    validEntity = True
+                    break
+            if not validEntity:
+                resultEntitySet.remove(entity)
+                self.allEntities.pop(entity)
+        return resultEntitySet
+
+    def parseConditions(self, conditionClauses: Union[None, list], entitiesPool) -> set:
+        """
+        conditionClauses:
+        [[("AND" | "OR" | None), ("Value Condition" | "Graph Condition"), (True | False), conditionValue], ...]
+
+        conditionValue:
+            if Value Condition:
+                [("ATTRIBUTE" | "RATTRIBUTE"), <User Input>,
+                ("EQ" | "CONTAINS" | "STARTSWITH" | "ENDSWITH" | "RMATCH"), <User Input>]
+            if Graph Condition:
+                [("CHILDOF" <ENTITY> | "DESCENDANTOF " <ENTITY> |
+                "PARENTOF" <ENTITY> | "ANCESTOROF " <ENTITY> |
+                "NUMCHILDREN" (" < " | " <= " | " > " | " >= " | " == ") <DIGITS> |
+                "NUMPARENTS" (" < " | " <= " | " > " | " >= " | " == ") <DIGITS> |
+                "CONNECTEDTO" <ENTITY> | "ISOLATED" | "ISROOT" | "ISLEAF")]
+        """
+
+        self.allEntities = {uid: self.allEntities[uid] for uid in self.allEntities if uid in entitiesPool}
+        uidsToSelect = set()
+
+        for conditionClause in conditionClauses:
+            isNot = conditionClause[2]
+            conditionValue = conditionClause[3]
+            try:
+                userInput1 = conditionValue[1]
+                userInput2 = conditionValue[3]
+            except IndexError:
+                # Not used in cases where a Graph Condition is specified
+                userInput1 = None
+                userInput2 = None
+            firstArgument = conditionValue[0]
+            if conditionClause[1] == "Value Condition":
+                matchingFields = []
+                if firstArgument == "ATTRIBUTE":
+                    if userInput1 in self.allEntityFields:
+                        matchingFields.append(userInput1)
+                else:
+                    try:
+                        attributeRegex = re.compile(userInput1)
+                    except re.error:
+                        continue
+                    for field in self.allEntityFields:
+                        if attributeRegex.match(field):
+                            matchingFields.append(field)
+
+                for matchingField in matchingFields:
+                    entitiesToRemove = []
+                    for entity in self.allEntities:
+                        attributeKeyValue = str(self.allEntities[entity].get(matchingField))
+                        if not self.checkVCHelper(conditionValue[2], isNot, attributeKeyValue, userInput2):
+                            if conditionClause[0] == "AND":
+                                entitiesToRemove.append(entity)
+                        else:
+                            uidsToSelect.add(entity)
+                    for entityToRemove in entitiesToRemove:
+                        uidsToSelect.remove(entityToRemove)
+
+            elif conditionClause[1] == "Graph Condition":
+                entitiesToRemove = []
+                for entity in self.allEntities:
+                    if not self.checkGCHelper(firstArgument, isNot, [entity] + conditionValue[1:]):
+                        if conditionClause[0] == "AND":
+                            entitiesToRemove.append(entity)
+                    else:
+                        uidsToSelect.add(entity)
+                for entityToRemove in entitiesToRemove:
+                    uidsToSelect.remove(entityToRemove)
+
+        uidsToRemove = set(self.allEntities).difference(uidsToSelect)
+        for entity in uidsToRemove:
+            self.allEntities.pop(entity, None)
+
+        return uidsToSelect
+
+    def canvasOr(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.union(canvasSetB)
+
+    def canvasAnd(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.intersection(canvasSetB)
+
+    def canvasAndNot(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.difference(canvasSetB)
+
+    def canvasOrNot(self, canvasSetA: set, canvasSetB: set, allEntitiesSet: set):
+        return canvasSetA.union(allEntitiesSet.difference(canvasSetB))
+
+    def checkEQ(self, valueA: str, valueB: str):
+        if valueA == valueB:
+            return True
+        return False
+
+    def checkContains(self, valueA: str, valueB: str):
+        if valueB in valueA:
+            return True
+        return False
+
+    def checkStartsWith(self, valueA: str, valueB: str):
+        if valueA.startswith(valueB):
+            return True
+        return False
+
+    def checkEndsWith(self, valueA: str, valueB: str):
+        if valueA.endswith(valueB):
+            return True
+        return False
+
+    def checkRMatch(self, valueA: str, valueB: str):
+        try:
+            valueMatch = re.compile(valueB)
+            if valueMatch.match(valueA):
+                return True
+        except re.error:
+            pass
+        return False
+
+    def checkVCHelper(self, checkType: str, isNot: bool, valueA: str, valueB: str):
+        returnVal = False
+        if checkType == "EQ":
+            returnVal = self.checkEQ(valueA, valueB)
+        elif checkType == "CONTAINS":
+            returnVal = self.checkContains(valueA, valueB)
+        elif checkType == "STARTSWITH":
+            returnVal = self.checkStartsWith(valueA, valueB)
+        elif checkType == "ENDSWITH":
+            returnVal = self.checkEndsWith(valueA, valueB)
+        elif checkType == "RMATCH":
+            returnVal = self.checkRMatch(valueA, valueB)
+        if isNot:
+            return not returnVal
+        return returnVal
+
+    def checkParentOf(self, valueA: str, valueB: str):
+        return self.databaseSnapshot.has_successor(valueA, valueB)
+
+    def checkAncestorOf(self, valueA: str, valueB: str):
+        try:
+            if valueB in nx.descendants(self.databaseSnapshot, valueA):
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkChildOf(self, valueA: str, valueB: str):
+        return self.databaseSnapshot.has_predecessor(valueA, valueB)
+
+    def checkDescendantOf(self, valueA: str, valueB: str):
+        try:
+            if valueB in nx.ancestors(self.databaseSnapshot, valueA):
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkNumChildren(self, valueA: str, valueB: str, valueC: int):
+        numChildren = len(list(self.databaseSnapshot.successors(valueA)))
+        returnValue = False
+        if valueB == "<":
+            if numChildren < valueC:
+                returnValue = True
+        elif valueB == "<=":
+            if numChildren <= valueC:
+                returnValue = True
+        elif valueB == ">":
+            if numChildren > valueC:
+                returnValue = True
+        elif valueB == ">=":
+            if numChildren >= valueC:
+                returnValue = True
+        elif valueB == "==":
+            if numChildren == valueC:
+                returnValue = True
+        return returnValue
+
+    def checkNumParents(self, valueA: str, valueB: str, valueC: int):
+        numParents = len(list(self.databaseSnapshot.predecessors(valueA)))
+        returnValue = False
+        if valueB == "<":
+            if numParents < valueC:
+                returnValue = True
+        elif valueB == "<=":
+            if numParents <= valueC:
+                returnValue = True
+        elif valueB == ">":
+            if numParents > valueC:
+                returnValue = True
+        elif valueB == ">=":
+            if numParents >= valueC:
+                returnValue = True
+        elif valueB == "==":
+            if numParents == valueC:
+                returnValue = True
+        return returnValue
+
+    def checkConnectedTo(self, valueA: str, valueB: str):
+        try:
+            if nx.has_path(self.databaseSnapshot, valueA, valueB):
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkIsolated(self, valueA: str):
+        try:
+            if valueA in self.databaseSnapshot.nodes and nx.is_isolate(self.databaseSnapshot, valueA):
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkIsRoot(self, valueA: str):
+        try:
+            if len(self.databaseSnapshot.in_edges(valueA)) == 0:
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkIsLeaf(self, valueA: str):
+        try:
+            if len(self.databaseSnapshot.out_edges(valueA)) == 0:
+                return True
+        except nx.NetworkXError:
+            pass
+        return False
+
+    def checkGCHelper(self, checkType: str, isNot: bool, args: list):
+        returnVal = False
+        if checkType == "CHILDOF":
+            returnVal = self.checkChildOf(*args)
+        elif checkType == "DESCENDANTOF":
+            returnVal = self.checkDescendantOf(*args)
+        elif checkType == "PARENTOF":
+            returnVal = self.checkParentOf(*args)
+        elif checkType == "ANCESTOROF":
+            returnVal = self.checkAncestorOf(*args)
+        elif checkType == "NUMCHILDREN":
+            returnVal = self.checkNumChildren(*args)
+        elif checkType == "NUMPARENTS":
+            returnVal = self.checkNumParents(*args)
+        elif checkType == "CONNECTEDTO":
+            returnVal = self.checkConnectedTo(*args)
+        elif checkType == "ISOLATED":
+            returnVal = self.checkIsolated(*args)
+        elif checkType == "ISROOT":
+            returnVal = self.checkIsRoot(*args)
+        elif checkType == "ISLEAF":
+            returnVal = self.checkIsLeaf(*args)
+        if isNot:
+            return not returnVal
+        return returnVal
+
+    def modifyNumify(self, valueA: str):
+        # Get the first number that shows up.
+        tempString = valueA.replace(',', '.')  # Making sure that floats are expressed the right way.
+        count = 0
+        for c in tempString:
+            if c not in string.digits:
+                count += 1
+            else:
+                break
+
+        count2 = 0
+        for c in tempString[count:]:
+            if c in string.digits or c == '.':
+                count2 += 1
+            else:
+                break
+
+        # If there are no numbers in the string, its numeric value is 0.
+        try:
+            floatValue = float(tempString[count:count + count2])
+        except ValueError:
+            floatValue = 0.0
+        return str(floatValue)
+
+    def modifyUpperCase(self, valueA: str):
+        return valueA.upper()
+
+    def modifyLowerCase(self, valueA: str):
+        return valueA.lower()
+
+    def parseModify(self, resultsToModify: (set, set), modifyQueries: list) -> (set, set):
+        """
+        modifyQueries:
+        [[("MODIFY" | "RMODIFY"), <User Input>, ("NUMIFY" | "UPPERCASE" | "LOWERCASE")], ...]
+        """
+
+        matchingFields = resultsToModify[1]
+
+        modifiedUIDs = set()
+        numifiedFields = set()
+
+        for modification in modifyQueries:
+            userInput1 = modification[1]
+            modificationType = modification[2]
+            modifyFields = []
+            if modification[0] == "MODIFY":
+                if userInput1 in resultsToModify[1]:
+                    modifyFields.append(userInput1)
+            else:
+                try:
+                    userInputRegex = re.compile(userInput1)
+                    modifyFields = [fieldMatch for fieldMatch in matchingFields if userInputRegex.match(fieldMatch)]
+                except (ValueError, re.error):
+                    continue
+            for entity in self.allEntities:
+                for modifyField in modifyFields:
+                    entityFieldValue = self.allEntities[entity].get(modifyField)
+                    if entityFieldValue is None:
+                        newFieldValue = None
+                    elif modificationType == "UPPERCASE":
+                        newFieldValue = self.modifyUpperCase(entityFieldValue)
+                    elif modificationType == "LOWERCASE":
+                        newFieldValue = self.modifyLowerCase(entityFieldValue)
+                    elif modificationType == "NUMIFY":
+                        newFieldValue = self.modifyNumify(entityFieldValue)
+                        numifiedFields.add(modifyField)
+                    else:
+                        newFieldValue = None
+                    if newFieldValue is not None:
+                        modifiedUIDs.add(entity)
+                        self.allEntities[entity][modifyField] = newFieldValue
+
+        return modifiedUIDs, numifiedFields
+
+    def parseQuery(self, selectClause: str, selectValue: Union[str, list], sourceClause: str,
+                   sourceValues: Union[None, list], conditionClauses: Union[None, list],
+                   modifyQueries: Union[list, None] = None) -> \
+            Optional[tuple[Optional[tuple[set, Union[set[Any], set[Union[str, Any]]]]],
+                           Optional[tuple[set[Any], set[Any]]]]]:
+
+        if self.databaseSnapshot is None:
+            return None
+
+        returnValue = None
+        modifications = None
+        fieldsToSelect = self.parseSelect(selectClause, selectValue)
+        if fieldsToSelect:
+            entitiesToConsider = self.parseSource(sourceClause, sourceValues, fieldsToSelect)
+            if entitiesToConsider:
+                if conditionClauses:
+                    entitiesToConsider = self.parseConditions(conditionClauses, entitiesToConsider)
+                returnValue = (entitiesToConsider, fieldsToSelect)
+                if modifyQueries:
+                    modifications = self.parseModify(returnValue, modifyQueries)
+
+        queryUID = str(uuid4())
+        self.QUERIES_HISTORY[queryUID] = (selectClause, selectValue, sourceClause, sourceValues, conditionClauses,
+                                          modifyQueries)
+
+        return returnValue, modifications

Core/MessageHandler.py

@@ -42,7 +42,7 @@ class MessageHandler:
                                           self.mainWindow.tr(message))
         return message
 
-    def error(self, message, popUp=True, exc_info=True):
+    def error(self, message, popUp=True, exc_info=False):
         self.linkScopeLogger.error(message, exc_info=exc_info)
         if popUp:
             msgBox = QtWidgets.QMessageBox()
@@ -101,7 +101,7 @@ class MessageHandler:
 
         # Do not handle root logger messages
         rootLogger = logging.getLogger()
-        for handler in rootLogger.handlers[:]:
+        for handler in rootLogger.handlers:
             if isinstance(handler, logging.Handler):
                 rootLogger.removeHandler(handler)
 

Core/ReportGeneration.py

@@ -16,6 +16,8 @@ from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
 from reportlab.lib.pagesizes import LETTER, inch
 from reportlab.graphics.shapes import Line, Drawing
 
+from Core.GlobalVariables import avoid_parsing_fields
+
 
 class MyDocTemplate(BaseDocTemplate):
     def __init__(self, filename, **kw):
@@ -256,7 +258,7 @@ class PDFReport:
                               ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
                               ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
         for key in list(entity):
-            if key != 'uid' and key != 'Icon' and key != 'Notes' and key != 'Date Last Edited':
+            if key not in avoid_parsing_fields and key != 'Notes':
                 valueText = entity[key]
                 value = "".join([valueText[counter:counter+80] + "\n"
                                  for counter in range(0, len(valueText), 80)])

Core/ResolutionManager.py

@@ -15,11 +15,11 @@ class ResolutionManager:
         self.mainWindow = mainWindow
         self.resolutions = {}
 
-    def loadResolutionsFromDir(self, directory: Path):
+    def loadResolutionsFromDir(self, directory: Path) -> None:
         exceptionsCount = 0
-        if self.resolutions.get(directory.stem) is None:
-            self.resolutions[directory.stem] = {}
         for resolution in listdir(directory):
+            resolution = str(resolution)
+            resolutionCategory = "Uncategorized"
             try:
                 if resolution.endswith('.py'):
                     resolutionName = resolution[:-3]
@@ -35,19 +35,29 @@ class ResolutionManager:
                     originTypes = resClassInst.originTypes
                     resultTypes = resClassInst.resultTypes
                     resolutionParameters = resClassInst.parameters
-                    self.resolutions[directory.stem][resNameString] = {'name': resNameString,
-                                                                       'description': resolutionDesc,
-                                                                       'originTypes': originTypes,
-                                                                       'resultTypes': resultTypes,
-                                                                       'parameters': resolutionParameters,
-                                                                       'resolution': resClass
-                                                                       }
+                    try:
+                        resolutionCategory = resClassInst.category
+                        if not isinstance(resolutionCategory, str):
+                            raise AttributeError()
+                    except AttributeError:
+                        pass
+                    if self.resolutions.get(resolutionCategory) is None:
+                        self.resolutions[resolutionCategory] = {}
+                    self.resolutions[resolutionCategory][resNameString] = {'name': resNameString,
+                                                                           'description': resolutionDesc,
+                                                                           'originTypes': originTypes,
+                                                                           'resultTypes': resultTypes,
+                                                                           'parameters': resolutionParameters,
+                                                                           'category': resolutionCategory,
+                                                                           'resolution': resClass
+                                                                           }
                     self.messageHandler.info("Loaded Resolution: " + resNameString)
             except Exception as e:
                 self.messageHandler.error("Cannot load resolutions from " + str(directory) + "\n Info: " + repr(e))
                 exceptionsCount += 1
-                if exceptionsCount > 5:
-                    self.messageHandler.critical("Failed loading too many modules to proceed.")
+                if exceptionsCount > 3:
+                    # Will not occur when loading modules with 3 or fewer resolutions, but that should be fine.
+                    self.messageHandler.critical("Failed loading too many resolutions to proceed.")
                     sys.exit(5)
 
     def getResolutionParameters(self, resolutionCategory, resolutionNameString):
@@ -73,22 +83,27 @@ class ResolutionManager:
                 return resolutionDescription
         return None
 
-    def loadResolutionsFromServer(self, serverRes):
-        self.resolutions |= serverRes
+    def loadResolutionsFromServer(self, serverRes) -> None:
+        for category in serverRes:
+            if category not in self.resolutions:
+                self.resolutions[category] = {}
+            for serverResolution in serverRes[category]:
+                self.resolutions[category][serverResolution] = serverRes[category][serverResolution]
 
-    def removeServerResolutions(self):
-        try:
-            self.resolutions.pop("Server Resolutions")
-        except KeyError:
-            pass
+    def removeServerResolutions(self) -> None:
+        for category in dict(self.resolutions):
+            for resolution in dict(self.resolutions[category]):
+                # If resolution class does not exist locally, then assume it exists on the server.
+                if self.resolutions[category][resolution]['resolution'] == '':
+                    self.resolutions[category].pop(resolution)
+        for category in dict(self.resolutions):
+            if len(self.resolutions[category]) == 0:
+                self.resolutions.pop(category)
 
-    def getResolutionCategories(self):
-        result = []
-        for category in self.resolutions:
-            result.append(category)
-        return result
+    def getResolutionCategories(self) -> list:
+        return list(self.resolutions)
 
-    def getResolutionsForEntityTypesByCategory(self, eTypes):
+    def getResolutionsForEntityTypesByCategory(self, eTypes) -> dict:
         """
         Gets a set of entity types, and returns a dictionary with all the resolutions that can take all
         included types as input.
@@ -103,12 +118,12 @@ class ResolutionManager:
                     result[category].append(resolution)
         return result
 
-    def getResolutionsInCategory(self, category):
+    def getResolutionsInCategory(self, category) -> list:
         if category in self.resolutions:
             return list(self.resolutions[category])
         return []
 
-    def getAllResolutions(self):
+    def getAllResolutions(self) -> list:
         categories = self.getResolutionCategories()
         result = []
         for category in categories:
@@ -120,10 +135,11 @@ class ResolutionManager:
         for category in self.resolutions:
             for resolution in self.resolutions[category]:
                 if self.resolutions[category][resolution]['name'] == resolutionName:
-                    if category == "Server Resolutions":
+                    # If resolution class does not exist locally, then assume it exists on the server.
+                    if self.resolutions[category][resolution]['resolution'] == '':
                         self.mainWindow.executeRemoteResolution(resolutionName, resolutionEntitiesInput, parameters,
                                                                 resolutionUID)
-                        # Returning a bool so we know that the resolution is running on the server.
+                        # Returning a bool, so we know that the resolution is running on the server.
                         return True
 
                     resolutionClass = self.resolutions[category][resolution]['resolution']()

Core/Resolutions/Core/ASNToCIDR.py

@@ -1,6 +1,7 @@
 class ASNToCIDR:
     # A string that is treated as the name of this resolution.
     name = "Get CIDR from ASN"
+    category = "Network Infrastructure"
 
     # A string that describes this resolution.
     description = "Get network information for an Autonomous System Number."

Core/Resolutions/Core/AffiliateCodesExtractor.py

@@ -13,6 +13,8 @@ class AffiliateCodesExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Affiliate Codes"
 
+    category = "Website Tracking"
+
     # A string that describes this resolution.
     description = "Returns Facebook and Amazon affiliate codes found in websites."
 
@@ -40,7 +42,7 @@ class AffiliateCodesExtractor:
                                            'value': {'Yes', 'No'}}}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
         import urllib
         import tldextract
@@ -121,6 +123,8 @@ class AffiliateCodesExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
                 return
 
@@ -158,6 +162,8 @@ class AffiliateCodesExtractor:
                                                 break
                                             except TimeoutError:
                                                 pass
+                                            except Error:
+                                                break
                                 else:
                                     GetAffiliateCodes(currentUID, newLink)
                             else:
@@ -174,6 +180,8 @@ class AffiliateCodesExtractor:
                                         break
                                     except TimeoutError:
                                         pass
+                                    except Error:
+                                        break
                             else:
                                 GetAffiliateCodes(currentUID, newLink)
 

Core/Resolutions/Core/CertificateInfo.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+
+
+class CertificateInfo:
+    name = "Analyze Website Certificate"
+    category = "Network Infrastructure"
+    description = "Get certificate information from the certificate of a certain website."
+    originTypes = {'Website'}
+    resultTypes = {'Phrase', 'Website', 'Domain', 'Country', 'Organization', 'Address', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import ssl
+        import socket
+
+        returnResults = []
+        sslContext = ssl.create_default_context()
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            websiteURL = entity['URL']
+            if websiteURL.startswith('https'):
+                addressWithoutHttps = websiteURL[8:].split('/')[0]
+            elif websiteURL.startswith('http'):
+                # Just in case an entity was created for the http version of a site,
+                #   but there is actually a https version of the site.
+                addressWithoutHttps = websiteURL[7:].split('/')[0]
+            else:
+                continue
+
+            with sslContext.wrap_socket(socket.socket(), server_hostname=addressWithoutHttps) as s:
+                try:
+                    s.connect((addressWithoutHttps, 443))
+                    websiteCertificate = s.getpeercert()
+                except socket.error:
+                    # If there's an error connecting, move on.
+                    continue
+
+            # Subject Details
+            streetAddr = None
+            locality = None
+            postalCode = None
+            subjectCountry = None
+            subjectSerial = None
+            subjectName = None
+            tentativeSubjectResults = []
+            for subjectAttributeOuter in websiteCertificate['subject']:
+                for subjectAttributeInner in subjectAttributeOuter:
+                    subjectAttributeInnerKey = subjectAttributeInner[0]
+                    subjectAttributeInnerValue = subjectAttributeInner[1]
+                    if subjectAttributeInnerKey == 'commonName':
+                        returnResults.append([{'Domain Name': subjectAttributeInnerValue,
+                                               'Entity Type': 'Domain'},
+                                              {uid: {'Resolution': 'Certificate Subject Common Name',
+                                                     'Notes': ''}}])
+
+                    elif subjectAttributeInnerKey == 'streetAddress':
+                        streetAddr = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'countryName':
+                        subjectCountry = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'postalCode':
+                        postalCode = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'localityName':
+                        locality = subjectAttributeInnerValue
+
+                    elif subjectAttributeInnerKey == 'serialNumber':
+                        subjectSerial = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'organizationName':
+                        subjectName = subjectAttributeInnerValue
+
+            subjectIndex = None
+            if subjectName is not None:
+                subjectIndex = len(returnResults)
+                subjectNameJSON = {'Organization Name': subjectName,
+                                   'Entity Type': 'Organization'}
+                if subjectSerial is not None:
+                    subjectNameJSON['Registration Number'] = subjectSerial
+                tentativeSubjectResults.append([subjectNameJSON,
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we only have the serial of the subject, use that as the subject entity representation.
+            elif subjectSerial is not None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Phrase': subjectSerial,
+                                                 'Entity Type': 'Phrase'},
+                                                {uid: {'Resolution': 'Certificate Subject Serial Number',
+                                                       'Notes': ''}}])
+
+            # If we have no company name and no serial, the certificate belongs to an unknown org.
+            if subjectIndex is None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Organization Name': 'Unknown Organization',
+                                                 'Entity Type': 'Organization'},
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we have an address, fill it in. If not, create separate entities for each element.
+            if streetAddr is not None:
+                streetAddressJSON = {'Street Address': streetAddr,
+                                     'Entity Type': 'Address'}
+                if subjectCountry is not None:
+                    streetAddressJSON['Country'] = subjectCountry
+                if postalCode is not None:
+                    streetAddressJSON['Postal Code'] = postalCode
+                if locality is not None:
+                    streetAddressJSON['Locality'] = locality
+                tentativeSubjectResults.append([streetAddressJSON,
+                                                {subjectIndex: {'Resolution': 'Certificate Subject Address',
+                                                                'Notes': ''}}])
+            else:
+                if subjectCountry is not None:
+                    tentativeSubjectResults.append([{'Country Name': subjectCountry,
+                                                     'Entity Type': 'Country'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Country',
+                                                                    'Notes': ''}}])
+                if postalCode is not None:
+                    tentativeSubjectResults.append([{'Phrase': postalCode,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Postal Code',
+                                                                    'Notes': ''}}])
+                if locality is not None:
+                    tentativeSubjectResults.append([{'Phrase': locality,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Locality',
+                                                                    'Notes': ''}}])
+            if len(tentativeSubjectResults) > 1:
+                # If we have more than just 'Unknown Organization' as a result for the subject, consider the output.
+                #   Otherwise, no point in including it.
+                returnResults += tentativeSubjectResults
+
+            # Validity dates for the certificate
+            returnResults.append([{'Date': websiteCertificate['notBefore'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Start Date',
+                                         'Notes': ''}}])
+            returnResults.append([{'Date': websiteCertificate['notAfter'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Expiry Date',
+                                         'Notes': ''}}])
+
+            # Domain names included in the certificate.
+            try:
+                for altNameAttribute in websiteCertificate['subjectAltName']:
+                    returnResults.append([{'Domain Name': altNameAttribute[1],
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Certificate Subject Alternate Name',
+                                                 'Notes': ''}}])
+            except KeyError:
+                pass
+
+            # OCSP URLs. Often just one.
+            try:
+                for ocsp in websiteCertificate['OCSP']:
+                    returnResults.append([{'URL': ocsp,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate OCSP URL',
+                                                 'Notes': ''}}])
+            except KeyError:
+                pass
+
+            # CA Issuer URL
+            try:
+                for caIssuer in websiteCertificate['caIssuers']:
+                    returnResults.append([{'URL': caIssuer,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Issuer URL',
+                                                 'Notes': ''}}])
+            except KeyError:
+                pass
+
+            # CRL URLs
+            try:
+                for crlDistributionPoint in websiteCertificate['crlDistributionPoints']:
+                    returnResults.append([{'URL': crlDistributionPoint,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Revocation List URL',
+                                                 'Notes': ''}}])
+            except KeyError:
+                pass
+
+            # Issuer information
+            orgName = None
+            orgCommonName = None
+            orgCountry = None
+            orgPostal = None
+            orgLocality = None
+            orgStateOrProvince = None
+            orgUnitName = None
+            for issuerAttributeOuter in websiteCertificate['issuer']:
+                for issuerAttributeInner in issuerAttributeOuter:
+                    issuerAttributeInnerKey = issuerAttributeInner[0]
+                    issuerAttributeInnerValue = issuerAttributeInner[1]
+                    if issuerAttributeInnerKey == 'organizationName':
+                        orgName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'commonName':
+                        orgCommonName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'countryName':
+                        orgCountry = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'postalCode':
+                        orgPostal = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'localityName':
+                        orgLocality = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'stateOrProvinceName':
+                        orgStateOrProvince = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'orgUnitName':
+                        orgUnitName = issuerAttributeInnerValue
+
+            issuerIndex = None
+            if orgName is not None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': orgName,
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Organization',
+                                             'Notes': ''}}])
+            if orgCommonName is not None:
+                if issuerIndex is not None:
+                    parentEntity = issuerIndex
+                else:
+                    parentEntity = uid
+                    issuerIndex = len(returnResults)
+                returnResults.append([{'Phrase': orgCommonName,
+                                       'Entity Type': 'Phrase'},
+                                      {parentEntity: {'Resolution': 'Certificate Issuer Common Name',
+                                                      'Notes': ''}}])
+            if issuerIndex is None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': 'Unknown Issuer',
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Name',
+                                             'Notes': ''}}])
+            if orgCountry is not None:
+                returnResults.append([{'Country Name': orgCountry,
+                                       'Entity Type': 'Country'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Country',
+                                                     'Notes': ''}}])
+            if orgPostal is not None:
+                returnResults.append([{'Phrase': orgPostal,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Postal Code',
+                                                     'Notes': ''}}])
+            if orgLocality is not None:
+                returnResults.append([{'Phrase': orgLocality,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Locality',
+                                                     'Notes': ''}}])
+            if orgStateOrProvince is not None:
+                returnResults.append([{'Phrase': orgStateOrProvince,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer State / Province',
+                                                     'Notes': ''}}])
+            if orgUnitName is not None:
+                returnResults.append([{'Phrase': orgUnitName,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Organizational Unit Name',
+                                                     'Notes': ''}}])
+
+            # Certificate info - serial number and certificate version.
+            returnResults.append([{'Phrase': websiteCertificate['serialNumber'],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Serial Number',
+                                         'Notes': ''}}])
+
+            returnResults.append([{'Phrase': 'Certificate Version: ' + str(websiteCertificate['version']),
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Version',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ContainsPhrase.py

@@ -5,6 +5,8 @@ class ContainsPhrase:
     # A string that is treated as the name of this resolution.
     name = "Contains Phrase"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
     description = "Checks if a Phrase exists in the Notes of a Phrase entity or the body of a Website."
 

Core/Resolutions/Core/DecodePhrase.py

@@ -2,6 +2,8 @@ class DecodePhrase:
     # A string that is treated as the name of this resolution.
     name = "Decode Phrase"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
     description = "Decode encoded phrases."
 

Core/Resolutions/Core/DomainFromPhrase.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+
+class DomainFromPhrase:
+    name = "Domain From Phrase"
+    category = "String Operations"
+    description = "Extract domain-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import tldextract
+
+        domainRegex = re.compile(
+            r'^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.'
+            r'([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$'
+        )
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase'].lower()
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if domainRegex.match(entityChunk):
+                    try:
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'Domain Name': entityChunk,
+                                                   'Entity Type': 'Domain'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Domain',
+                                                                   'Notes': ''}}])
+                    except Exception:
+                        pass
+
+        return returnResults

Core/Resolutions/Core/EmailExtractor.py

@@ -5,6 +5,8 @@ class EmailExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Emails"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns the email addresses present on a website or index page of a domain."
 
@@ -12,18 +14,7 @@ class EmailExtractor:
 
     resultTypes = {'Email Address'}
 
-    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
-                                               'explored to discover more entities. Each entity discovered after '
-                                               'exploring sites linked in the original website or domain is said to '
-                                               'have a "depth" value of 1. Entities found from exploring the links on '
-                                               'this page would have a "depth" of 2, and so on. A larger value could '
-                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
-                                               'The default value is "0", which means only the provided website, or '
-                                               'the index page of the domain provided, is explored.',
-                                'type': 'String',
-                                'value': '0',
-                                'default': '0'},
-                  'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
+    parameters = {'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
                                                'code of the website. However, not all emails on the site may exist in '
                                                'that format. Using Regex can result in more emails being extracted, '
                                                'however it is possible that some false positives may be extracted '
@@ -44,9 +35,8 @@ class EmailExtractor:
                   }
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
-        import tldextract
         import re
         from email_validator import validate_email, caching_resolver, EmailNotValidError
 
@@ -66,12 +56,10 @@ class EmailExtractor:
         resolver = caching_resolver(timeout=10)
         verifyDomain = True if parameters['Verify Email Domain Validity'] == 'Yes' else False
 
-        exploredDepth = set()
-
         # The software can deduplicate, but handling it here is better.
         allEmails = set()
 
-        def extractEmails(currentUID: str, site: str, depth: int):
+        def extractEmails(currentUID: str, site: str):
             page = context.new_page()
             pageResolved = False
             for _ in range(3):
@@ -81,11 +69,34 @@ class EmailExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
             soupContents = BeautifulSoup(page.content(), 'lxml')
             if useRegex:
-                potentialEmails = emailRegex.findall(soupContents.get_text())
+                # Remove <span> and <noscript> tags, and attempt some basic de-obfuscation.
+                while True:
+                    try:
+                        soupContents.noscript.extract()
+                    except AttributeError:
+                        break
+                while True:
+                    try:
+                        soupContents.span.extract()
+                    except AttributeError:
+                        break
+                siteContent = soupContents.get_text()
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*at\s*(\]|\>|\))+\s*', '@', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*dot\s*(\]|\>|\))+\s*', '.', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*\.\s*(\]|\>|\))+\s*', '.', siteContent)
+
+                potentialEmails = emailRegex.findall(siteContent)
                 for potentialEmail in potentialEmails:
                     try:
                         valid = validate_email(potentialEmail, dns_resolver=resolver, check_deliverability=verifyDomain)
@@ -113,39 +124,23 @@ class EmailExtractor:
                                                                     'Notes': ''}}])
                         except EmailNotValidError:
                             pass
-                    elif newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractEmails(currentUID, newLink, newDepth)
-
-            linksInLinkHref = soupContents.find_all('link')
-            for tag in linksInLinkHref:
-                newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractEmails(currentUID, newLink, newDepth)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch()
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
             )
             for entity in entityJsonList:
                 uid = entity['uid']
-                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
                     url = 'http://' + url
-                domain = tldextract.extract(url).fqdn
-                extractEmails(uid, url, maxDepth)
+                extractEmails(uid, url)
             browser.close()
 
         return returnResults

Core/Resolutions/Core/EmailToDomain.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+
+class EmailToDomain:
+    name = "Email To Domain"
+    category = "Network Infrastructure"
+    description = "Get the Domain that an email address belongs to."
+    originTypes = {'Email Address'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            # There is no provider that I am aware of that allows '@' signs in the user part of the email.
+            try:
+                returnResults.append([{'Domain Name': primaryField.split('@')[1].strip(),
+                                       'Entity Type': 'Domain'},
+                                      {entity['uid']: {'Resolution': 'Email To Domain',
+                                                       'Notes': ''}}])
+            except Exception:
+                pass
+
+        return returnResults

Core/Resolutions/Core/EmailToPhrase.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+
+class EmailToPhrase:
+    name = "Email Username To Phrase"
+    category = "String Operations"
+    description = "Get the username associated with the given email address."
+    originTypes = {'Email Address'}
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            returnResults.append([{'Phrase': primaryField.split('@')[0],
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'Email Username To Phrase',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ExtractDOCXMeta.py

@@ -4,6 +4,7 @@
 class ExtractDOCXMeta:
     # A string that is treated as the name of this resolution.
     name = "Get DOCX Metadata"
+    category = "File Operations"
 
     # A string that describes this resolution.
     description = "Returns a set of nodes that contain all the metadata info of a docx file."

Core/Resolutions/Core/ExtractDateCreated.py

@@ -3,6 +3,7 @@
 
 class ExtractDateCreated:
     name = "Extract Date Created"
+    category = "Date Operations"
     description = "Extract the date in the Date Created field of the selected entities."
     originTypes = {'*'}
     resultTypes = {'Date'}

Core/Resolutions/Core/ExtractPDFMeta.py

@@ -4,6 +4,7 @@
 class ExtractPDFMeta:
     # A string that is treated as the name of this resolution.
     name = "Get PDF Metadata"
+    category = "File Operations"
 
     # A string that describes this resolution.
     description = "Returns a set of nodes that contain all the metadata info of pdf files."

Core/Resolutions/Core/FileExtractor.py

@@ -19,6 +19,8 @@ class FileExtractor:
     # A string that is treated as the name of this resolution.
     name = "Find Hosted File URLs"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns Nodes of files in websites and domains."
 
@@ -45,7 +47,7 @@ class FileExtractor:
         from binascii import hexlify
         from pathlib import Path
         from bs4 import BeautifulSoup
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
 
         try:
             maxDepth = max(int(parameters['Max Depth']), 0)
@@ -74,6 +76,8 @@ class FileExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
 
             soupContents = BeautifulSoup(page.content(), 'lxml')
 

Core/Resolutions/Core/FileHasher.py

@@ -3,6 +3,7 @@
 
 class FileHasher:
     name = "Get File Hash"
+    category = "File Operations"
     description = "Get the Hash of a file."
     originTypes = {"Image", "Document", "Video", "Archive", "Disk"}
     resultTypes = {'Hash'}

Core/Resolutions/Core/GetExternalURLs.py

@@ -18,9 +18,10 @@ Tweaks to this script can be made depending on each investigator's workflow.
 
 class GetExternalURLs:
     name = "Get External Urls"
+    category = "Website Information"
     description = "Returns all links to external sites on a website."
     originTypes = {'Website'}
-    resultTypes = {'Website'}
+    resultTypes = {'Website', 'Onion Website'}
 
     parameters = {'Element types to check': {'description': 'Select the types of elements to investigate for '
                                                             'external links. Note that "a" elements have the lowest '
@@ -35,11 +36,12 @@ class GetExternalURLs:
 
     def resolution(self, entityJsonList, parameters):
         import tldextract
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
         import re
-        import urllib
+        import urllib.parse
 
+        onionRegex = re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$""")
         returnResult = []
 
         extract_a = '<a> elements' in parameters['Element types to check']
@@ -48,24 +50,24 @@ class GetExternalURLs:
 
         # Sites like youtube replace external links with a redirect link originating
         #   from the site itself. This sort of gets around that.
-        redirectRegex = re.compile(r'q=[^\s][^&^#]*', re.IGNORECASE)
+        redirectRegex = re.compile(r'\?.*(q|url)=\S[^&#?]+', re.IGNORECASE)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch()
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
             )
-            allPages = []
             page = context.new_page()
+            externalUrls = {}
 
             for site in entityJsonList:
                 uid = site['uid']
                 url = site['URL']
-                if url is None:
+                parsedURL = urllib.parse.urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
                     continue
-                if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
                 domain = tldextract.extract(url).fqdn
 
                 # Try to load the page a few times, in case of timeouts.
@@ -73,50 +75,81 @@ class GetExternalURLs:
                 for _ in range(3):
                     try:
                         page.goto(url, wait_until="networkidle", timeout=10000)
-                        allPages.append((domain, page, uid))
+                        soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                        if extract_a:
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]):
+                                    if domain not in link:
+                                        newLink = link.split('#')[0].split('?')[0]
+                                        try:
+                                            externalUrls[newLink].add(uid)
+                                        except KeyError:
+                                            externalUrls[newLink] = {uid}
+                                    else:
+                                        redirectLinks = redirectRegex.findall(link)
+                                        if 'redirect' in link and len(redirectLinks) > 0:
+                                            try:
+                                                newLink = str(urllib.parse.unquote(redirectLinks[0]))[2:]
+                                                try:
+                                                    externalUrls[newLink].add(uid)
+                                                except KeyError:
+                                                    externalUrls[newLink] = {uid}
+                                            except IndexError:
+                                                try:
+                                                    externalUrls[redirectLinks[0]].add(uid)
+                                                except KeyError:
+                                                    externalUrls[redirectLinks[0]] = {uid}
+                                            except Exception:
+                                                pass
+
+                        if extract_img:
+                            linksInImgSrc = soupContents.find_all('img')
+                            for tag in linksInImgSrc:
+                                link = tag.get('src', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]):
+                                    if domain not in link:
+                                        newLink = link.split('#')[0].split('?')[0]
+                                        try:
+                                            externalUrls[newLink].add(uid)
+                                        except KeyError:
+                                            externalUrls[newLink] = {uid}
+
+                        if extract_link:
+                            linksInLinkHref = soupContents.find_all('link')
+                            for tag in linksInLinkHref:
+                                link = tag.get('href', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]):
+                                    if domain not in link:
+                                        newLink = link.split('#')[0].split('?')[0]
+                                        try:
+                                            externalUrls[newLink].add(uid)
+                                        except KeyError:
+                                            externalUrls[newLink] = {uid}
                         break
                     except TimeoutError:
                         pass
-            for urlVisited in allPages:
-                externalUrls = set()
-                soupContents = BeautifulSoup(urlVisited[1].content(), 'lxml')
+                    except Error:
+                        break
 
-                if extract_a:
-                    linksInAHref = soupContents.find_all('a')
-                    for tag in linksInAHref:
-                        link = tag.get('href', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-                                else:
-                                    redirLinks = redirectRegex.findall(link)
-                                    if 'redirect' in link and len(redirLinks) > 0:
-                                        newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
-                                        externalUrls.add(newLink)
-
-                if extract_img:
-                    linksInImgSrc = soupContents.find_all('img')
-                    for tag in linksInImgSrc:
-                        link = tag.get('src', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-
-                if extract_link:
-                    linksInLinkHref = soupContents.find_all('link')
-                    for tag in linksInLinkHref:
-                        link = tag.get('href', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-
-                for externalUrl in externalUrls:
-                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
-                                         {urlVisited[2]: {'Resolution': 'External Link', 'Notes': ''}}])
             page.close()
+            context.close()
             browser.close()
 
+        for externalUrl in externalUrls:
+            onionCheck = onionRegex.findall(externalUrl)
+            if len(onionCheck) == 1:
+                for urlUid in externalUrls[externalUrl]:
+                    returnResult.append([{'Onion URL': externalUrl, 'Entity Type': 'Onion Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+            else:
+                for urlUid in externalUrls[externalUrl]:
+                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+
         return returnResult

Core/Resolutions/Core/GetInternalURLs.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+
+
+class GetInternalURLs:
+    name = "Get Domain Urls"
+    category = "Website Information"
+    description = "Returns all URLs in a Domain."
+    originTypes = {'Website', 'Domain'}
+    resultTypes = {'Website'}
+
+    parameters = {'Include Resources': {'description': 'Should the resolution also return links to images and internal '
+                                                       'links (i.e. links to resources used by the site)? Note that '
+                                                       'doing so will make the resolution take more time to complete.',
+                                        'type': 'SingleChoice',
+                                        'value': {'Include images and internal links', 'Only consider links to pages'},
+                                        'default': 'Only consider links to pages'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import urllib.parse
+
+        returnResult = []
+        internalUrls = {}
+
+        considerResources = False if parameters['Include Resources'] == 'Only consider links to pages' else True
+
+        def handleLink(currentLink, currentUrl, currentDomain):
+            if currentLink.startswith('//'):
+                if currentUrl.endswith('/'):
+                    currentUrl = currentUrl[:-1]
+                currentLink = currentUrl + currentLink[1:]
+            elif currentLink.startswith('/'):
+                urlParts = urllib.parse.urlparse(currentUrl)
+                currentLink = urlParts.scheme + '://' + urlParts.netloc + currentLink
+            parsedCurrentURL = urllib.parse.urlparse(link)
+            if all([parsedCurrentURL.scheme, parsedCurrentURL.netloc]):
+                if currentDomain in currentLink:
+                    newLink = currentLink.split('#')[0].split('?')[0]
+                    if newLink.endswith('/'):
+                        newLink = newLink[:-1]
+                    return newLink
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                entityType = site['Entity Type']
+                if entityType == 'Domain':
+                    url = 'http://' + site['Domain Name']
+                else:
+                    url = site['URL']
+                parsedURL = urllib.parse.urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+                domain = tldextract.extract(url).fqdn
+
+                domainUrls = [url]
+                index = 0
+                while index < len(domainUrls):
+                    url = domainUrls[index]
+                    index += 1
+                    for _ in range(3):
+                        try:
+                            page.goto(url, wait_until="networkidle", timeout=10000)
+                            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                            if domain not in page.url:
+                                # Handle redirects to other sites
+                                try:
+                                    internalUrls[page.url].add(uid)
+                                except KeyError:
+                                    internalUrls[page.url] = {uid}
+                                break
+
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                handledLink = handleLink(link, url, domain)
+                                if handledLink:
+                                    try:
+                                        internalUrls[handledLink].add(uid)
+                                    except KeyError:
+                                        internalUrls[handledLink] = {uid}
+                                    if handledLink not in domainUrls:
+                                        domainUrls.append(handledLink)
+
+                            if considerResources:
+                                linksInImgSrc = soupContents.find_all('img')
+                                for tag in linksInImgSrc:
+                                    link = tag.get('src', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+
+                                linksInLinkHref = soupContents.find_all('link')
+                                for tag in linksInLinkHref:
+                                    link = tag.get('href', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+                            break
+                        except TimeoutError:
+                            pass
+                        except Error:
+                            break
+
+            page.close()
+            context.close()
+            browser.close()
+
+        for internalUrl in internalUrls:
+            for urlUid in internalUrls[internalUrl]:
+                returnResult.append([{'URL': internalUrl, 'Entity Type': 'Website'},
+                                     {urlUid: {'Resolution': 'Internal Link', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/GetWebsiteBody.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetWebsiteBody:
-    # A string that is treated as the name of this resolution.
-    name = "Get Website Body"
-
-    # A string that describes this resolution.
-    description = "Returns the contents of the body tag of the selected websites."
-
-    originTypes = {'Website'}
-
-    resultTypes = {'Phrase'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-
-        headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
-        }
-
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-
-            primaryField = entity['URL']
-
-            if primaryField.startswith('http://') or primaryField.startswith('https://'):
-                url = primaryField
-            else:
-                url = 'http://' + primaryField
-
-            r = requests.get(url, headers=headers)
-            doc = r.text
-
-            returnResults.append([{'Phrase': 'Website Body: ' + primaryField,
-                                   'Notes': doc,
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Website Body', 'Notes': ''}}])
-        return returnResults

Core/Resolutions/Core/GetWebsiteText.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+
+class GetWebsiteText:
+    # A string that is treated as the name of this resolution.
+    name = "Get Website Text"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns the text content of the selected websites."
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from bs4 import BeautifulSoup
+        from bs4.element import Comment
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from urllib.parse import urlparse
+
+        returnResults = []
+
+        def tag_visible(element):
+            if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
+                return False
+            if isinstance(element, Comment):
+                return False
+            return True
+
+        def text_from_html(body):
+            soup = BeautifulSoup(body, 'html.parser')
+            texts = soup.findAll(text=True)
+            visible_texts = filter(tag_visible, texts)
+            return u" ".join(t.strip() for t in visible_texts if t.strip() != '')
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                parsedURL = urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+
+                # Try to load the page a few times, in case of timeouts.
+                # I don't think making parts of this async actually helps in this case.
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        textContent = text_from_html(page.content())
+                        returnResults.append([{'Phrase': 'Website Body of: ' + url,
+                                               'Notes': textContent,
+                                               'Entity Type': 'Phrase'},
+                                              {uid: {'Resolution': 'Website Body', 'Notes': ''}}])
+                        break
+                    except TimeoutError:
+                        pass
+                    except Error:
+                        break
+
+        return returnResults

Core/Resolutions/Core/HostnameToDomain.py

@@ -2,8 +2,9 @@
 
 
 class HostnameToDomain:
-    name = "Hostname to Domain"
-    description = "Find the domain of any given hostname."
+    name = "Get Top Level Domain"
+    category = "Network Infrastructure"
+    description = "Get the top level domain of any given website or subdomain."
     originTypes = {'Domain', 'Website'}
     resultTypes = {'Domain'}
     parameters = {}

Core/Resolutions/Core/HostnameToIP.py

@@ -3,6 +3,7 @@
 
 class HostnameToIP:
     name = "Hostname To IP"
+    category = "Network Infrastructure"
     description = "Gets the IP associated with the given hostname."
     originTypes = {'Domain', 'Website'}
     resultTypes = {'IP Address', 'IPv6 Address'}

Core/Resolutions/Core/IPToASN.py

@@ -2,6 +2,8 @@ class IPToASN:
     # A string that is treated as the name of this resolution.
     name = "Get ASN From IP"
 
+    category = "Network Infrastructure"
+
     # A string that describes this resolution.
     description = "Get the Autonomous System Number that the selected IP Addresses belong to."
 

Core/Resolutions/Core/IPToHostname.py

@@ -3,6 +3,7 @@
 
 class IPToHostname:
     name = "IP To Hostname"
+    category = "Network Infrastructure"
     description = "Gets the Fully Qualified Domain Name associated with the given IPv4 / IPv6 address."
     originTypes = {'IPv6 Address', 'IP Address'}
     resultTypes = {'Domain'}

Core/Resolutions/Core/IPWhois.py

@@ -3,6 +3,7 @@
 
 class IPWhois:
     name = "IPv4 WhoIs Information"
+    category = "Network Infrastructure"
     description = "Extract information from the Whois record that corresponds to a given IP Address."
     originTypes = {'IP Address'}
     resultTypes = {'Country', 'Autonomous System', 'Email Address'}

Core/Resolutions/Core/ImageToDevice.py

@@ -3,6 +3,7 @@
 
 class ImageToDevice:
     name = "Device From Image"
+    category = "File Operations"
     description = "Find information about on what device an image was taken with."
     originTypes = {"Image"}
     resultTypes = {'Phrase'}

Core/Resolutions/Core/ImageToGeoLocation.py

@@ -3,6 +3,7 @@
 
 class ImageToGeoLocation:
     name = "Geolocation From Image"
+    category = "File Operations"
     description = "Find information about where an image was taken."
     originTypes = {"Image"}
     resultTypes = {'GeoCoordinates'}

Core/Resolutions/Core/JSCodeExtractor.py

@@ -5,6 +5,8 @@ class JSCodeExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Tracking Codes"
 
+    category = "Website Tracking"
+
     # A string that describes this resolution.
     description = "Returns Nodes of 'ca-pub', 'ua' and 'gtm' tracking codes for websites and/or domains."
 
@@ -15,16 +17,35 @@ class JSCodeExtractor:
     parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright
+        from playwright.sync_api import sync_playwright, Error
+        from base64 import b64decode
         import re
         returnResults = []
         requestUrlsParsed = set()
 
         uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
         pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
-        gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
-        gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
-        qualtricsRegex = re.compile(r'\bQ_ZID=[a-zA-Z_0-9]*\b', re.IGNORECASE)
+        gtmRegex = re.compile(r'\bGTM-[A-Z\d]{1,7}\b')
+        gRegex = re.compile(r'\bG-[A-Z\d]{1,15}\b', re.IGNORECASE)
+        qualtricsRegex = re.compile(r'\bQ_(?:Z|S)ID=\w*\b')
+        pingdomRegex = re.compile(r'\bpa-[a-fA-F\d]{24}.js\b$')
+        mPulseRegex = re.compile(r'go-mpulse.net/boomerang/[A-Z\d]{5}(?:-[A-Z\d]{5}){4}\b')
+        contextWebRegex = re.compile(r'\.contextweb\.com.*token=.*')
+        facebookRegex = re.compile(r'facebook.com/tr?.*id=\d*')
+        googleMapsRegex = re.compile(r'maps\.googleapis\.com/maps/api/js\?.*client=[ a-zA-Z\d-]*')
+        marketoRegex = re.compile(r'marketo\.com/rtp-api/v1/rtp.js\?.*aid=[ a-zA-Z\d-]*')
+        visualWebsiteOptimizerRegex = re.compile(r'visualwebsiteoptimizer\.com/j\.php\?.*a=\d*')
+        optimizeRegex = re.compile(r'googleoptimize\.com/optimize\.js\?.*id=[A-Z\d-]*')
+        markMonitorRegex = re.compile(r'\.adsrvr\.org/track/evnt/\?.*adv=[a-zA-Z\d-]*')
+        zendeskRegex = re.compile(r'\.zdassets\.com/ekr/snippet\.js\?.*key=[a-zA-Z\d-]*')
+        quantServeRegex = re.compile(r'pixel\.quantserve\.com/pixel/.*\.gif\?')
+        cookieLawRegex = re.compile(r'cdn\.cookielaw\.org/consent/.*/')
+        oneTagRegex = re.compile(r'get\.s-onetag\.com/.*/')
+        bounceExchangeRegex = re.compile(r'tag\.bounceexchange\.com/.*/')
+        pushlyRegex = re.compile(r'cdn.p-n.io/.*domain_key=[\w%]*')
+        akamaiRegex = re.compile(r'/akam/.*a=[\w%=]*')
+        demdexRegex = re.compile(r'dpm\.demdex\.net/id\?.*d_orgid=[^&]*')
+        brightcoveRegex = re.compile(r'metrics\.brightcove\.com/.*/tracker\?.*&account=[^&]*')
 
         def GetTrackingCodes(pageUid, requestUrl) -> None:
             if requestUrl not in requestUrlsParsed:
@@ -54,14 +75,119 @@ class JSCodeExtractor:
                                            'Entity Type': 'Phrase'},
                                           {pageUid: {'Resolution': 'Qualtrics Tracking Code',
                                                      'Notes': ''}}])
+                for pCode in pingdomRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': pCode[:-3],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Pingdom Tracking Code',
+                                                     'Notes': ''}}])
+                for mCode in mPulseRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': mCode.split('/')[-1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'mPulse Tracking Code',
+                                                     'Notes': ''}}])
+                for cCode in contextWebRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': cCode.split('token=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'ContextWeb Tracking Code',
+                                                     'Notes': ''}}])
+                for fCode in facebookRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': fCode.split('id=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Facebook Tracking Pixel Code',
+                                                     'Notes': ''}}])
+                for mapsCode in googleMapsRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': mapsCode.split('client=', 1)[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google Maps Client Code',
+                                                     'Notes': ''}}])
+                for marketoCode in marketoRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': marketoCode.split('aid=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Marketo Tracking Code',
+                                                     'Notes': ''}}])
+                for vwoCode in visualWebsiteOptimizerRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': vwoCode.split('a=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Visual Website Optimizer Tracking User ID',
+                                                     'Notes': ''}}])
+                for oCode in optimizeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': oCode.split('id=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google Optimize ID',
+                                                     'Notes': ''}}])
+                for mmCode in markMonitorRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': mmCode.split('adv=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Mark Monitor Tracking ID',
+                                                     'Notes': ''}}])
+                for zCode in zendeskRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': zCode.split('key=')[1].split('&')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Zendesk ID',
+                                                     'Notes': ''}}])
+                for qsCode in quantServeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': qsCode.split('/pixel/')[1].split('.gif')[0],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'QuantServe Tracking Pixel ID',
+                                                     'Notes': ''}}])
+                for clCode in cookieLawRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': clCode.split('/')[2],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'CookieLaw Website ID',
+                                                     'Notes': ''}}])
+                for otCode in oneTagRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': otCode.split('/')[1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'OneTag Tracking ID',
+                                                     'Notes': ''}}])
+                for beCode in bounceExchangeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': beCode.split('/')[1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'BounceExchange Tracking ID',
+                                                     'Notes': ''}}])
+                for pushlyCode in pushlyRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': pushlyCode.split('domain_key=')[1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Pushly Website ID',
+                                                     'Notes': ''}}])
+                for aCode in akamaiRegex.findall(requestUrl):
+                    encodedTracking = aCode.split('a=', 1)[1]
+                    encodedTracking = encodedTracking.replace('%3D', '=')
+                    decodedTracking = b64decode(encodedTracking).decode('utf-8').split('t=')[1].split('&')[0]
+                    returnResults.append([{'Phrase': decodedTracking,
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Akamai Website ID',
+                                                     'Notes': 'SHA-1 Sum'}}])
+                for dCode in demdexRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': dCode.split('d_orgid=')[1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'DemDex (Adobe) Website ID',
+                                                     'Notes': ''}}])
+                for bCode in brightcoveRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': bCode.split('account=')[1],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'BrightCove Website ID',
+                                                     'Notes': ''}}])
 
         with sync_playwright() as p:
             browser = p.firefox.launch()
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'
             )
-            page = context.new_page()
+            contextNoJS = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
+                java_script_enabled=False
+            )
+            pageJS = context.new_page()
+            pageNoJS = contextNoJS.new_page()
+            uid = None
+
+            # Subscribe to "request" events.
+            pageJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+            pageNoJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+
             for site in entityJsonList:
                 uid = site['uid']
                 url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
@@ -70,10 +196,16 @@ class JSCodeExtractor:
                 if not url.startswith('http://') and not url.startswith('https://'):
                     url = 'http://' + url
 
-                # Subscribe to "request" events.
-                page.on("request", lambda request: GetTrackingCodes(uid, request.url))
-                page.goto(url)
-            page.close()
+                try:
+                    pageJS.goto(url, wait_until="networkidle")
+                except Error:
+                    pass
+                try:
+                    pageNoJS.goto(url, wait_until="networkidle")
+                except Error:
+                    pass
+            pageJS.close()
+            pageNoJS.close()
             browser.close()
 
         return returnResults

Core/Resolutions/Core/PhoneNumbersExtractor.py

@@ -10,6 +10,8 @@ class PhoneNumbersExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Phone Numbers"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns the Phone Numbers present on a website or index page of a domain."
 
@@ -17,34 +19,19 @@ class PhoneNumbersExtractor:
 
     resultTypes = {'Phone Number'}
 
-    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
-                                               'explored to discover more entities. Each entity discovered after '
-                                               'exploring sites linked in the original website or domain is said to '
-                                               'have a "depth" value of 1. Entities found from exploring the links on '
-                                               'this page would have a "depth" of 2, and so on. A larger value could '
-                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
-                                               'The default value is "0", which means only the provided website, or '
-                                               'the index page of the domain provided, is explored.',
-                                'type': 'String',
-                                'value': '0',
-                                'default': '0'}}
+    parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
-        import tldextract
+        import re
+
+        cleanTagsRegex = re.compile(r'<.*?>')
+        phoneNumCharsExclusion = re.compile(r'[^ -+()\[\]\d]')
 
         returnResults = []
 
-        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
-        try:
-            maxDepth = max(int(parameters['Max Depth']), 0)
-        except ValueError:
-            return "Invalid value provided for Max Webpages to follow."
-
-        exploredDepth = set()
-
-        def extractTels(currentUID: str, site: str, depth: int):
+        def extractTels(currentUID: str, site: str):
             page = context.new_page()
             pageResolved = False
             for _ in range(3):
@@ -54,8 +41,14 @@ class PhoneNumbersExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
 
             soupContents = BeautifulSoup(page.content(), 'lxml')
 
@@ -68,39 +61,35 @@ class PhoneNumbersExtractor:
                                                'Entity Type': 'Phone Number'},
                                               {currentUID: {'Resolution': 'Phone Number Found',
                                                             'Notes': ''}}])
-                    elif newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractTels(currentUID, newLink, newDepth)
 
-            linksInLinkHref = soupContents.find_all('link')
-            for tag in linksInLinkHref:
-                newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractTels(currentUID, newLink, newDepth)
+            textTags = soupContents.find_all('p')
+            for tag in textTags:
+                tagContents = tag.text
+                cleanTagContentsList = re.sub(cleanTagsRegex, '', tagContents).split('\n')
+                for cleanTag in cleanTagContentsList:
+                    cleanTag = cleanTag.strip()
+                    if not phoneNumCharsExclusion.findall(cleanTag) and len(re.findall(r'\d', cleanTag)) >= 3:
+                        returnResults.append([{'Phone Number': cleanTag,
+                                               'Entity Type': 'Phone Number'},
+                                              {currentUID: {'Resolution': 'Phone Number Found',
+                                                            'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch()
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36',
             )
             for entity in entityJsonList:
                 uid = entity['uid']
-                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
                     url = 'http://' + url
-                domain = tldextract.extract(url).fqdn
-                extractTels(uid, url, maxDepth)
+                extractTels(uid, url)
             browser.close()
 
         return returnResults

Core/Resolutions/Core/PhraseSimilarity.py

@@ -2,6 +2,7 @@
 
 class PhraseSimilarity:
     name = "String Similarity Check"
+    category = "String Operations"
     description = "Find the ratio of similarity between two strings."
     originTypes = {'*'}
     resultTypes = {'Phrase'}

Core/Resolutions/Core/RegexMatch.py

@@ -5,6 +5,8 @@ class RegexMatch:
     # A string that is treated as the name of this resolution.
     name = "Get Regex Match"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
     description = "Extract text from Phrases and Websites with Regular Expressions."
 

Core/Resolutions/Core/TikTokVideoPublishDetails.py

@@ -5,6 +5,8 @@ class TikTokVideoPublishDetails:
     # A string that is treated as the name of this resolution.
     name = "TikTok Video Publishing Details"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Extracts information from public TikTok Videos."
 

Core/Resolutions/Core/ToPhrase.py

@@ -3,6 +3,7 @@
 
 class ToPhrase:
     name = "Convert To Phrase"
+    category = "String Operations"
     description = "Convert the primary field of selected entities to a Phrase entity."
     originTypes = {'*'}
     resultTypes = {'Phrase'}

Core/Resolutions/Core/WebsiteFromPhrase.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+
+class WebsiteFromPhrase:
+    name = "Website From Phrase"
+    category = "String Operations"
+    description = "Extract website-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import tldextract
+
+        websiteRegex = re.compile(r"""^https?://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$""")
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase']
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if websiteRegex.match(entityChunk):
+                    try:
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'URL': entityChunk,
+                                                   'Entity Type': 'Website'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Website',
+                                                                   'Notes': ''}}])
+                    except Exception:
+                        pass
+
+        return returnResults

Core/Resolutions/Core/WordCounter.py

@@ -3,6 +3,7 @@
 
 class WordCounter:
     name = "Get Number Of Words"
+    category = "String Operations"
     description = "Count the number of words in a given phrase."
     originTypes = {'Phrase'}
     resultTypes = {'Phrase'}

Core/ResourceHandler.py

@@ -14,8 +14,10 @@ from ast import literal_eval
 from base64 import b64decode
 from dateutil import parser
 
-from PySide6.QtGui import QIcon
-from PySide6.QtCore import QByteArray
+from PySide6.QtCore import QByteArray, QSize, QUrl, Qt
+from PySide6 import QtWidgets, QtGui
+
+from Core.Interface import Stylesheets
 
 
 class ResourceHandler:
@@ -59,6 +61,8 @@ class ResourceHandler:
                                       "Resources" / "Icons" / "DrawLink.png"),
                       "rearrange": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
                                        "Resources" / "Icons" / "RearrangeGraph.png"),
+                      "colorPicker": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                         "Resources" / "Icons" / "ColorPicker.png"),
                       }
         # These are not meant to be strict - just restrictive enough such that users don't put in utter nonsense.
         # Note that regex isn't always the best way of validating fields, but it should be good enough for our
@@ -66,7 +70,8 @@ class ResourceHandler:
         self.checks = {'Email': re.compile(r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"""),
                        'Phonenumber': re.compile(r"""^(\+|00)?[0-9\(\) \-]{3,32}$"""),
                        'String': re.compile(r""".+"""),
-                       'URL': re.compile(r"""^https?://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$"""),
+                       'URL': re.compile(r"""^(?:(?:http|ftp)s?|file)://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$"""),
+                       'Onion': re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$"""),
                        'Domain': re.compile(r"""^(\S(?<!\.)(?!/)(?<!/)){1,63}(\.(\S(?<!\.)(?!/)(?<!/)){1,63})+$"""),
                        'Float': re.compile(r"""^([-+])?(\d|\.(?=\d))+$"""),
                        'WordString': re.compile(r"""^\D+$"""),
@@ -160,6 +165,9 @@ class ResourceHandler:
         """
         Check that the attribute value given matches the regex of the category 'check'.
         """
+        # Ignore checks - this is used for non-string attributes in special entities.
+        if check == 'None':
+            return True
         attrCheck = self.checks.get(check)
         if attrCheck is None:
             return False
@@ -351,11 +359,11 @@ class ResourceHandler:
 
     def getLinkPicture(self):
         picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Resolution.png"
-        return QIcon(str(picture)).pixmap(40, 40)
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
 
     def getLinkArrowPicture(self):
         picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Right-Arrow.svg"
-        return QIcon(str(picture)).pixmap(40, 40)
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
 
     def deconstructGraph(self, graph: nx.DiGraph) -> tuple:
         nodes = {}
@@ -410,3 +418,171 @@ class ResourceHandler:
             returnGraph.add_edge(*edgeUID, **graphEdges[edge])
 
         return returnGraph
+
+
+class StringPropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(StringPropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+
+    def getValue(self):
+        return self.text()
+
+
+class FilePropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(FilePropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+        self.fileDialog = QtWidgets.QFileDialog()
+
+    def getValue(self):
+        return self.text()
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        fileChosen = self.fileDialog.getOpenFileName(self,
+                                                     "Open File",
+                                                     str(Path.home()),
+                                                     options=QtWidgets.QFileDialog.DontUseNativeDialog)
+        self.setText(fileChosen[0])
+
+
+class SingleChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOption):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(SingleChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOption is None:
+            defaultOption = ''
+
+        for option in enforceOptionsSet:
+            radioButton = QtWidgets.QRadioButton(option)
+            radioButton.setStyleSheet(Stylesheets.RADIO_BUTTON_STYLESHEET)
+            if option == defaultOption:
+                radioButton.setChecked(True)
+            else:
+                radioButton.setChecked(False)
+            self.options.append(radioButton)
+            vboxLayout.addWidget(radioButton)
+
+    def getValue(self):
+        for option in self.options:
+            if option.isChecked():
+                return option.text()
+
+        return ''
+
+
+class MultiChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOptions):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(MultiChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOptions is None:
+            defaultOptions = []
+
+        for option in enforceOptionsSet:
+            checkBox = QtWidgets.QCheckBox(option)
+            checkBox.setStyleSheet(Stylesheets.CHECK_BOX_STYLESHEET)
+            if option in defaultOptions:
+                checkBox.setChecked(True)
+            else:
+                checkBox.setChecked(False)
+            self.options.append(checkBox)
+            vboxLayout.addWidget(checkBox)
+
+    def getValue(self):
+        valuesSelected = []
+        for option in self.options:
+            if option.isChecked():
+                valuesSelected.append(option.text())
+
+        return valuesSelected
+
+
+class MinSizeStackedLayout(QtWidgets.QStackedLayout):
+    """
+    Resize the layout to always take up the appropriate space for the currently selected widget.
+    Otherwise, large widgets (due to selecting entities with long strings of text) will stretch
+    out the ScrollArea and make the other, non-selected widgets to look bad when the layout
+    switches over.
+
+    https://stackoverflow.com/a/34300567
+    """
+    def sizeHint(self) -> QSize:
+        return self.currentWidget().sizeHint()
+
+    def minimumSize(self) -> QSize:
+        return self.currentWidget().minimumSize()
+
+
+class RichNotesEditor(QtWidgets.QTextBrowser):
+
+    def __init__(self, parent=None, currentText: str = '#### Type notes here.\n', allowEditing: bool = True):
+        super(RichNotesEditor, self).__init__(parent=parent)
+        self.allowEditing = allowEditing
+        self.setReadOnly(True)
+        if allowEditing:
+            self.setUndoRedoEnabled(True)
+            self.setTextInteractionFlags(Qt.TextBrowserInteraction | Qt.TextSelectableByKeyboard)
+        else:
+            self.setUndoRedoEnabled(False)
+            self.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.TextSelectableByKeyboard
+                                         | Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard)
+
+        self.contents = currentText
+        self.setMarkdown(self.contents)
+        self.textFormat = self.currentCharFormat()
+
+    def startEditing(self) -> None:
+        if self.allowEditing:
+            if self.isReadOnly():
+                # Reset char format to plain text.
+                self.setCurrentCharFormat(self.textFormat)
+                self.setPlainText(self.contents)
+                self.setReadOnly(False)
+
+    def stopEditing(self) -> None:
+        if not self.isReadOnly():
+            self.contents = self.toPlainText()
+            self.setMarkdown(self.contents)
+            self.setReadOnly(True)
+
+    def dropEvent(self, e: QtGui.QDropEvent) -> None:
+        if self.allowEditing:
+            editingBefore = self.isReadOnly()
+            self.startEditing()
+            super(RichNotesEditor, self).dropEvent(e)
+            if editingBefore:
+                self.stopEditing()
+
+    def mousePressEvent(self, ev: QtGui.QMouseEvent) -> None:
+        potentialLink = self.anchorAt(ev.pos())
+        if not potentialLink:
+            if ev.button() == QtGui.Qt.LeftButton:
+                self.startEditing()
+        super(RichNotesEditor, self).mousePressEvent(ev)
+
+    def focusOutEvent(self, ev: QtGui.QFocusEvent) -> None:
+        if not self.underMouse():
+            if self.isActiveWindow():
+                self.stopEditing()
+        super(RichNotesEditor, self).focusOutEvent(ev)
+
+    def doSetSource(self, name: Union[QUrl, str], resourceType: QtGui.QTextDocument.ResourceType = ...) -> None:
+        QtGui.QDesktopServices.openUrl(name)

Core/SettingsObject.py

@@ -23,6 +23,15 @@ class SettingsObject(dict):
         super().__init__()
         self.setValue("Program/BaseDir", "Unset")  # dirname(abspath(getsourcefile(lambda:0))) + "/../" )
         self.setValue("Program/GraphLayout", "dot")
+        self.setValue("Program/Internal/Macros", "")
+        self.setValue("Program/Graphics/EntityTextFontType", "Mono")
+        self.setValue("Program/Graphics/EntityTextFontSize", "11")
+        self.setValue("Program/Graphics/EntityTextFontBoldness", "700")
+        self.setValue("Program/Graphics/LinkTextFontType", "Mono")
+        self.setValue("Program/Graphics/LinkTextFontSize", "11")
+        self.setValue("Program/Graphics/LinkTextFontBoldness", "700")
+        self.setValue("Program/Graphics/EntityTextColor", "#000000")  # RGB
+        self.setValue("Program/Graphics/LinkTextColor", "#000000")  # RGB
         self.setValue("Project/Name", "Untitled")
         self.setValue("Project/BaseDir", "")
         self.setValue("Project/FilesDir", "")
@@ -35,6 +44,7 @@ class SettingsObject(dict):
         self.setValue("Project/Question Answering Retriever Value", "10")
         self.setValue("Project/Question Answering Reader Value", "10")
         self.setValue("Project/Server/Project", "")
+        self.setValue("Project/Server/Collectors", "{}")
 
         # The value '20' equates to logging.INFO
         # It's not necessary to set this, but we will for

Core/URLManager.py

@@ -7,6 +7,7 @@ from os import symlink
 from shutil import copy2
 from hashlib import sha3_512
 from binascii import hexlify
+from urllib.parse import urlparse
 
 from PySide6 import QtCore
 
@@ -30,19 +31,15 @@ class URLManager:
         Takes a list of QUrls and returns a list of entities that correspond
         to them.
         """
-        if len(urls) == 1:
-            url = urls[0]
-            entityJson = self.handleURL(url)
-            return [entityJson]
-
         returnValue = []
         for url in urls:
-            returnValue += [self.handleURL(url)]
+            returnValue.append(self.handleURL(url))
 
         return returnValue
 
     def handleURL(self, url):
-        if not url.isValid():
+        parsedURL = urlparse(url.toString())
+        if not url.isValid() or not all([parsedURL.scheme, parsedURL.netloc]):
             return None
         if url.isLocalFile():
             return self.handleLocalURL(url)
@@ -50,7 +47,7 @@ class URLManager:
             return self.handleRemoteURL(url)
 
     def handleURLString(self, urlString):
-        self.handleURL(QtCore.QUrl(urlString))
+        return self.handleURL(QtCore.QUrl(urlString))
 
     def handleLocalURL(self, url):
         urlPath = Path(url.toLocalFile())
@@ -105,5 +102,9 @@ class URLManager:
         return savePath
 
     def handleRemoteURL(self, url):
-        entity = {'Entity Type': 'Website', 'URL': url.toString()}
+        stringURL = url.toString()
+        if self.mainWindow.RESOURCEHANDLER.runCheckOnAttribute(stringURL, 'Onion'):
+            entity = {'Entity Type': 'Onion Website', 'Onion URL': stringURL}
+        else:
+            entity = {'Entity Type': 'Website', 'URL': stringURL}
         return entity

Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py

@@ -3,6 +3,7 @@
 
 class AircraftInquiryByDealer:
     name = "Aircraft Inquiry By Dealer"
+    category = "Aircraft"
     description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
     originTypes = {"Company"}
     resultTypes = {'Phrase', 'Company'}

Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py

@@ -3,6 +3,7 @@
 
 class AircraftInquiryByEngine:
     name = "Aircraft Inquiry By Engine"
+    category = "Aircraft"
     description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
     originTypes = {"Phrase"}
     resultTypes = {"Phrase"}

Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py

@@ -3,6 +3,7 @@
 
 class AircraftInquiryByNNumber:
     name = "Aircraft Inquiry By N-Number"
+    category = "Aircraft"
     description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
     originTypes = {"Phrase"}
     resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}

Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py

@@ -3,6 +3,7 @@
 
 class AircraftInquiryByPersonName:
     name = "Aircraft Inquiry By Person Name"
+    category = "Aircraft"
     description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
     originTypes = {"Person"}
     resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}

Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py

@@ -2,7 +2,8 @@
 
 
 class AircraftInquiryBySerialNumber:
-    name = "AircraftInquiry By Serial Number"
+    name = "Aircraft Inquiry By Serial Number"
+    category = "Aircraft"
     description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
     originTypes = {"Identification Number"}
     resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}

Modules/Aleph/Aleph_Entity_Search.py

@@ -3,6 +3,7 @@
 
 class Aleph_Entity_Search:
     name = "Aleph Entity Search"
+    category = "Aleph OCCRP"
     description = "Find information about a given search parameter"
     originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
     resultTypes = {'Phrase'}

Modules/Aleph/GetCollectionByID.py

@@ -3,6 +3,7 @@
 
 class GetCollectionByID:
     name = "Get Collections By ID"
+    category = "Aleph OCCRP"
     description = "Find information about Collections and their IDs"
     originTypes = {'Phrase'}
     resultTypes = {'Phrase. Person, Address, Phone Number, Email Address, Country, Bank Account'}

Modules/Aleph/GetCollectionsInfo.py

@@ -3,6 +3,7 @@
 
 class GetCollectionsInfo:
     name = "Get Collections Info"
+    category = "Aleph OCCRP"
     description = "Find information about Collections and their IDs"
     originTypes = {'Phrase'}
     resultTypes = {'Phrase, Aleph ID'}

Modules/Aleph/GetSimilarEntities.py

@@ -3,6 +3,7 @@
 
 class GetSimilarEntities:
     name = "Get Similar Entities"
+    category = "Aleph OCCRP"
     description = "Find information about similar entities"
     originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
     resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID'}

Modules/Amass/Amass_Domain.py

@@ -3,121 +3,174 @@
 
 class Amass_Domain:
     name = "Amass Domain Scan"
+    category = "Network Infrastructure"
     description = "Find information about a particular domain"
     originTypes = {'Domain'}
-    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain'}
-    parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://virustotal.com.',
+    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain', 'IPv6 Address'}
+    parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://virustotal.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'AlienVault': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://otx.alienvault.com.',
                                  'type': 'String',
                                  'value': 'None',
                                  'global': True,
                                  'default': 'None'},
-                  'AlienVault': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://otx.alienvault.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
                   'BinaryEdge': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://app.binaryedge.com.',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'C99': {'description': 'Enter your api key under your profile after'
                                          ' signing up on https://c99.nl.',
                           'type': 'String',
-                          'value': 'None'},
+                          'value': 'None',
+                          'global': True,
+                          'default': 'None'},
                   'Censys': {'description': 'Enter your api key under your profile after'
                                             ' signing up on https://censys.io.',
                              'type': 'String',
-                             'value': 'None'},
+                             'value': 'None',
+                             'global': True,
+                             'default': 'None'},
                   'Chaos': {'description': 'Enter your api key under your profile after'
                                            ' signing up on https://chaos.projectdiscovery.io.',
                             'type': 'String',
-                            'value': 'None'},
+                            'value': 'None',
+                            'global': True,
+                            'default': 'None'},
                   'Cloudflare': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://cloudflare.com.',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'DNSDB': {'description': 'Enter your api key under your profile after'
                                            ' signing up on https://dnsdb.info.',
                             'type': 'String',
-                            'value': 'None'},
+                            'value': 'None',
+                            'global': True,
+                            'default': 'None'},
                   'GitHub': {'description': 'Enter your api key under your profile after'
                                             ' signing up on https://github.com.',
                              'type': 'String',
-                             'value': 'None'},
+                             'value': 'None',
+                             'global': True,
+                             'default': 'None'},
                   'Hunter': {'description': 'Enter your api key under your profile after'
                                             ' signing up on https://hunter.io.',
                              'type': 'String',
-                             'value': 'None'},
+                             'value': 'None',
+                             'global': True,
+                             'default': 'None'},
                   'IPinfo': {'description': 'Enter your api key under your profile after'
                                             ' signing up on https://ipinfo.io.',
                              'type': 'String',
-                             'value': 'None'},
+                             'value': 'None',
+                             'global': True,
+                             'default': 'None'},
                   'NetworksDB': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://networksdb.io.',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'PassiveTotal': {'description': 'Enter your api key under your profile after'
                                                   ' signing up on https://passivetotal.com .',
                                    'type': 'String',
-                                   'value': 'None'},
+                                   'value': 'None',
+                                   'global': True,
+                                   'default': 'None'},
                   'ReconDev': {'description': 'Enter your api key under your profile after'
                                               ' signing up on https://recon.dev.',
                                'type': 'String',
-                               'value': 'None'},
+                               'value': 'None',
+                               'global': True,
+                               'default': 'None'},
                   'SecurityTrails': {'description': 'Enter your api key under your profile after'
                                                     ' signing up on https://securitytrails.com.',
                                      'type': 'String',
-                                     'value': 'None'},
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
                   'Shodan': {'description': 'Enter your api key under your profile after'
                                             ' signing up on https://shodan.io.',
                              'type': 'String',
-                             'value': 'None'},
+                             'value': 'None',
+                             'global': True,
+                             'default': 'None'},
                   'Spyse': {'description': 'Enter your api key under your profile after'
                                            ' signing up on https://spyse.com.',
                             'type': 'String',
-                            'value': 'None'},
+                            'value': 'None',
+                            'global': True,
+                            'default': 'None'},
                   'ThreatBook': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://threatbook.cn.',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'Umbrella': {'description': 'Enter your api key under your profile after'
                                               ' signing up on https://umbrella.cisco.com.',
                                'type': 'String',
-                               'value': 'None'},
+                               'value': 'None',
+                               'global': True,
+                               'default': 'None'},
                   'URLScan': {'description': 'Enter your api key under your profile after'
                                              ' signing up on https://urlscan.io.',
                               'type': 'String',
-                              'value': 'None'},
+                              'value': 'None',
+                              'global': True,
+                              'default': 'None'},
                   'WhoisXMLAPI': {'description': 'Enter your api key under your profile after'
                                                  ' signing up on https://whoisxmlapi.com.',
                                   'type': 'String',
-                                  'value': 'None'},
+                                  'value': 'None',
+                                  'global': True,
+                                  'default': 'None'},
                   'ZETAlytics': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://zetalytics.com.',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'ZoomEye': {'description': 'Please Enter the Username and password with a space '
                                              'in between',
                               'type': 'String',
-                              'value': 'None'},
+                              'value': 'None',
+                              'global': True,
+                              'default': 'None'},
                   'FacebookCT': {'description': 'Please Enter the api key and secret with a space '
                                                 'in between. Obtain them at https://developer.facebook.com',
                                  'type': 'String',
-                                 'value': 'None'},
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
                   'Twitter': {'description': 'Please Enter the api key and secret with a space '
                                              'in between. Obtain them at https://developer.twitter.com',
                               'type': 'String',
-                              'value': 'None'},
+                              'value': 'None',
+                              'global': True,
+                              'default': 'None'},
                   'ReconDev.free': {
                       'description':
                           'Please Enter the api key under your profile after signing up on https://recon.dev',
                       'type': 'String',
-                      'value': 'None'},
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'},
                   'ReconDev.paid': {
                       'description':
                           'Please Enter the api key under your profile after signing up on https://recon.dev',
                       'type': 'String',
-                      'value': 'None'}}
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'}}
 
     def resolution(self, entityJsonList, parameters):
         from pathlib import Path
@@ -128,107 +181,100 @@ class Amass_Domain:
         from docker.errors import APIError
 
         return_result = []
-        client = docker.from_env()
         # Generate Config as a temporary file:
-        for entity in entityJsonList:
-            primary_field = entity[list(entity)[1]].strip()
-            try:
-                with tempfile.TemporaryDirectory() as tempDir:
-                    tempPath = Path(tempDir).absolute()
-                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                         suffix='Config',
-                                                         dir=tempPath)
-                    config.write("share = true\n")
-                    config.write("[scope]\n")
-                    config.write("port = 80\n")
-                    config.write("port = 443\n")
-                    config.write("[data_sources]\n")
-                    config.write("minimum_ttl = 1440\n")
-                    for parameter in parameters:
-                        if parameters[f'{parameter}'] != 'None':
-                            field1 = f"[data_sources.{parameter}]"
-                            field2 = f"[data_sources.{parameter}.Credentials]"
-                            if parameter == "ZoomEye":
-                                username, password = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"username = {username}\n")
-                                config.write(f"password = {password}\n")
-                            elif parameter == "FacebookCT":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.app1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "Twitter":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.account1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "ReconDev.paid":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.paid\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            elif parameter == "ReconDev.free":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.free\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            else:
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            path_to_config = config.name
-                            path_to_config = path_to_config.replace(str(tempPath), "")
-                            config.seek(0)
-                            container = client.containers.run("caffix/amass:latest",
-                                                              f"enum -src -d {primary_field} "
-                                                              f"-config /.config/amass{path_to_config}",
-                                                              volumes={
-                                                                  str(tempPath): {'bind': '/.config/amass',
-                                                                                  'mode': 'rw'}},
-                                                              remove=True)
-                        else:
-                            container = client.containers.run("caffix/amass:latest",
-                                                              f"enum -src -d {primary_field}",
-                                                              volumes={
-                                                                  str(tempPath): {'bind': '/.config/amass',
-                                                                                  'mode': 'rw'}},
-                                                              remove=True)
+        with tempfile.TemporaryDirectory() as tempDir:
+            tempPath = Path(tempDir).absolute()
+            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                 suffix='Config',
+                                                 dir=tempPath)
+            config.write("share = true\n")
+            config.write("[scope]\n")
+            config.write("port = 80\n")
+            config.write("port = 443\n")
+            config.write("[data_sources]\n")
+            config.write("minimum_ttl = 1440\n")
+            for parameter in self.parameters:
+                if parameters[f'{parameter}'] != 'None':
+                    field1 = f"[data_sources.{parameter}]"
+                    field2 = f"[data_sources.{parameter}.Credentials]"
+                    if parameter == "ZoomEye":
+                        username, password = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"username = {username}\n")
+                        config.write(f"password = {password}\n")
+                    elif parameter == "FacebookCT":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.app1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "Twitter":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.account1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "ReconDev.paid":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.paid\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    elif parameter == "ReconDev.free":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.free\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    else:
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+            path_to_config = Path(config.name).name
+            config.seek(0)
+            for entity in entityJsonList:
+                primary_field = entity["Domain Name"].strip()
+                try:
+                    client = docker.from_env()
+                    container = client.containers.run("caffix/amass:latest",
+                                                      f"enum -src -d {primary_field} "
+                                                      f"-config /.config/amass/{path_to_config}",
+                                                      volumes={
+                                                          str(tempPath): {'bind': '/.config/amass',
+                                                                          'mode': 'rw'}},
+                                                      remove=True)
                     jsonFile = tempPath / 'amass.json'
+                    jsonContents = ""
                     if jsonFile.exists():
-                        jsonContents = ""
                         with open(jsonFile, 'r') as jsonFileHandler:
                             jsonContents = jsonFileHandler.read()
-            except (APIError, docker.errors.ContainerError) as error:
-                return "Something happened to docker - Cannot continue"
-            uid = entity['uid']
-            for dictionary in jsonContents.splitlines():
-                index_of_child = len(return_result)
-                line_dictionary = json.loads(dictionary)
-                size = len(line_dictionary['addresses'])
-                return_result.append([{'Domain Name': str(line_dictionary['name']),
-                                       'Entity Type': 'Domain'},
-                                      {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
-                for ip in range(size):
-                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
-                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IP Address'},
-                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
-                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
-                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IPv6 Address'},
-                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
-                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
-                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
-                                           'Entity Type': 'Autonomous System'},
-                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
-                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Amass Domain Scan Description',
-                                                            'Notes': ''}}])
-        jsonFileHandler.close()
+                    client.close()
+                except (APIError, docker.errors.ContainerError) as error:
+                    return "Something happened to the docker container - Cannot continue: " + str(error)
+                uid = entity['uid']
+                for dictionary in jsonContents.splitlines():
+                    index_of_child = len(return_result)
+                    line_dictionary = json.loads(dictionary)
+                    size = len(line_dictionary['addresses'])
+                    return_result.append([{'Domain Name': str(line_dictionary['name']),
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
+                    for ip in range(size):
+                        if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
+                            return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
+                                                   'Entity Type': 'IP Address'},
+                                                  {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
+                        elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
+                            return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
+                                                   'Entity Type': 'IPv6 Address'},
+                                                  {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
+                        return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
+                                               'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
+                                               'Entity Type': 'Autonomous System'},
+                                              {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
+                        return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
+                                               'Entity Type': 'Phrase'},
+                                              {index_of_child: {'Resolution': 'Amass Domain Scan Description',
+                                                                'Notes': ''}}])
+            config.close()
         return return_result

Modules/Amass/Amass_Intel.py

@@ -3,9 +3,10 @@
 
 class Amass_Intel:
     name = "Amass Intel Scan"
+    category = "Network Infrastructure"
     description = "Find information about a particular domain"
-    originTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
-    resultTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
+    originTypes = {'Domain', 'IP Address', 'Autonomous System'}
+    resultTypes = {'Domain'}
     parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
                                                 ' signing up on https://virustotal.com.',
                                  'type': 'String',
@@ -126,168 +127,105 @@ class Amass_Intel:
         from docker.errors import APIError
 
         return_result = []
-        client = docker.from_env()
         # Generate Config as a temporary file:
-        for entity in entityJsonList:
-            primary_field = entity[list(entity)[1]].strip()
-            try:
-                with tempfile.TemporaryDirectory() as tempDir:
-                    tempPath = Path(tempDir).absolute()
-                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                         suffix='Config',
-                                                         dir=tempPath)
-                    config.write("share = true\n")
-                    config.write("[scope]\n")
-                    config.write("port = 80\n")
-                    config.write("port = 443\n")
-                    config.write("[data_sources]\n")
-                    config.write("minimum_ttl = 1440\n")
-                    for parameter in parameters:
-                        if parameters[f'{parameter}'] != 'None':
-                            field1 = f"[data_sources.{parameter}]"
-                            field2 = f"[data_sources.{parameter}.Credentials]"
-                            if parameter == "ZoomEye":
-                                username, password = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"username = {username}\n")
-                                config.write(f"password = {password}\n")
-                            elif parameter == "FacebookCT":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.app1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "Twitter":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.account1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "ReconDev.paid":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.paid\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            elif parameter == "ReconDev.free":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.free\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            else:
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            path_to_config = config.name
-                            path_to_config = path_to_config.replace(str(tempPath), "")
-                            config.seek(0)
-                            # print(config.read())
-                            # print(path_to_config)
-                            # print(entity['Entity Type'])
-                            if entity['Entity Type'] == "Domain":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -d {primary_field} -config /.config/amass"
-                                                                  f"{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
-                                    'Entity Type'] == "Company":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -org {primary_field} -whois -config "
-                                                                  f"/.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "IP Address":
-                                try:
-                                    ip_address(primary_field)
-                                except ValueError:
-                                    return "The Entity Provided isn't a valid IP Address"
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -addr {primary_field} -whois -config "
-                                                                  f"/.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Autonomous System":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}   "
-                                                                  f" -whois -config /.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                        else:
-                            if entity['Entity Type'] == "Domain":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -d {primary_field}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
-                                    'Entity Type'] == "Company":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -org {primary_field} -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "IP Address":
-                                try:
-                                    ip_address(primary_field)
-                                except ValueError:
-                                    return "The Entity Provided isn't a valid IP Address"
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -addr {primary_field} -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Autonomous System":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}"
-                                                                  f" -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                    jsonFile = tempPath / 'amass.json'
-                    if jsonFile.exists():
-                        jsonContents = ""
-                        with open(jsonFile, 'r') as jsonFileHandler:
-                            jsonContents = jsonFileHandler.read()
-            except (APIError, docker.errors.ContainerError) as error:
-                return "Soomething happened to docker - Cannot continue."
-            uid = entity['uid']
-            for dictionary in jsonContents.splitlines():
-                index_of_child = len(return_result)
-                line_dictionary = json.loads(dictionary)
-                size = len(line_dictionary['addresses'])
-                return_result.append([{'Domain Name': str(line_dictionary['name']),
-                                       'Entity Type': 'Domain'},
-                                      {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
-                for ip in range(size):
-                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
-                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IP Address'},
-                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
-                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
-                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IPv6 Address'},
-                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
-                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
-                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
-                                           'Entity Type': 'Autonomous System'},
-                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
-                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Amass Intel Scan Description',
-                                                            'Notes': ''}}])
-        jsonFileHandler.close()
+        with tempfile.TemporaryDirectory() as tempDir:
+            tempPath = Path(tempDir).absolute()
+            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                 suffix='Config',
+                                                 dir=tempPath)
+            config.write("share = true\n")
+            config.write("[scope]\n")
+            config.write("port = 80\n")
+            config.write("port = 443\n")
+            config.write("[data_sources]\n")
+            config.write("minimum_ttl = 1440\n")
+            for parameter in parameters:
+                if parameters[f'{parameter}'] != 'None':
+                    field1 = f"[data_sources.{parameter}]"
+                    field2 = f"[data_sources.{parameter}.Credentials]"
+                    if parameter == "ZoomEye":
+                        username, password = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"username = {username}\n")
+                        config.write(f"password = {password}\n")
+                    elif parameter == "FacebookCT":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.app1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "Twitter":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.account1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "ReconDev.paid":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.paid\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    elif parameter == "ReconDev.free":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.free\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    else:
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+            path_to_config = "/" + Path(config.name).name
+            config.seek(0)
+            for entity in entityJsonList:
+                primary_field = entity[list(entity)[1]].strip()
+                try:
+                    client = docker.from_env()
+                    if entity['Entity Type'] == "Domain":
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -whois -d {primary_field} -config /.config/amass"
+                                                          f"{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    elif entity['Entity Type'] == "IP Address":
+                        try:
+                            ip_address(primary_field)
+                        except ValueError:
+                            return "The Entity Provided isn't a valid IP Address"
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -addr {primary_field} -config "
+                                                          f"/.config/amass{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    elif entity['Entity Type'] == "Autonomous System":
+                        if primary_field.startswith('AS'):
+                            primary_field = primary_field[2:]
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -asn {primary_field}"
+                                                          f" -config /.config/amass{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    textFile = tempPath / 'amass.txt'
+                    textContents = ""
+                    if textFile.exists():
+                        with open(textFile, 'r') as textFileHandler:
+                            textContents = textFileHandler.read()
+
+                    client.close()
+                except (APIError, docker.errors.ContainerError) as error:
+                    return "Something happened to the docker container - Cannot continue: " + str(error)
+                uid = entity['uid']
+                for newDomain in textContents.splitlines():
+                    return_result.append([{'Domain Name': newDomain.strip(),
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
+
+            config.close()
         return return_result

Modules/BigMatch/BigMatch.py

@@ -3,6 +3,7 @@
 
 class BigMatch:
     name = "BigMatch Search"
+    category = "Secrets & Leaks"
     description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
     originTypes = {"Image", "Document", "Archive"}
     resultTypes = {'Website'}
@@ -10,7 +11,7 @@ class BigMatch:
 
     def resolution(self, entityJsonList, parameters):
         from pathlib import Path
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
 
         return_result = []
@@ -59,6 +60,8 @@ class BigMatch:
                         break
                     except TimeoutError:
                         pass
+                    except Error:
+                        break
             page.close()
             browser.close()
         return return_result

Modules/BinaryEdge/BinaryEdgeHost.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+
+class BinaryEdgeHost:
+    name = "BinaryEdge Host Query"
+    category = "Network Infrastructure"
+    description = "Get information about a host from BinaryEdge."
+    originTypes = {"IP Address", "IPv6 Address"}
+    resultTypes = {'Port'}
+    parameters = {'BinaryEdge API Key': {'description': "Enter your BinaryEdge API key. Sign up for one at "
+                                                        "https://www.binaryedge.io/",
+                                         'type': 'String',
+                                         'value': '',
+                                         'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+
+        baseURL = 'https://api.binaryedge.io/v2/query/ip/'
+        requestHeaders = {'X-Key': parameters['BinaryEdge API Key'].strip()}
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if entity['Entity Type'] == 'IP Address':
+                primaryField = entity['IP Address']
+            elif entity['Entity Type'] == 'IPv6 Address':
+                primaryField = entity['IPv6 Address']
+            else:
+                continue
+            infoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
+            statusCode = infoRequest.status_code
+
+            if statusCode == 401:
+                return "The BinaryEdge API key provided is not valid."
+            elif statusCode == 403:
+                return "The BinaryEdge API key provided does not have permission to access this resource."
+            elif statusCode != 200:
+                continue
+            requestContent = json.loads(infoRequest.content)
+
+            for event in requestContent['events']:
+                for result in event['results']:
+                    originDetails = result['origin']
+                    targetDetails = result['target']
+                    resultDetails = result['result']
+                    if 'state' not in resultDetails['data']:
+                        # Discard return result if it doesn't actually give us useful info about the state of the port.
+                        # This happens in cases where the API returns stuff like the ciphers used in an SSH service.
+                        #   There seems to always be a result with the simple port info, so we will use that one.
+                        continue
+
+                    returnResults.append([{'Port': targetDetails['ip'] + ':' + str(targetDetails['port']) + ':' +
+                                                   targetDetails['protocol'],
+                                           'State': resultDetails['data']['state']['state'],
+                                           'Banner': resultDetails['data']['service'].get('banner', 'N/A'),
+                                           'Product': resultDetails['data']['service'].get('product', 'Unknown'),
+                                           'Entity Type': 'Port'},
+                                          {uid: {'Resolution': 'BinaryEdge Scan Timestamp: ' + str(originDetails['ts']),
+                                                 'Notes': ''}}])
+
+        return returnResults

Modules/BreachData/HIBP.xml

@@ -0,0 +1,34 @@
+<HIBP>
+    <Data_Breach>
+        <Attributes>
+            <Attribute default="Breach Name" check="String" primary="True">Breach Name</Attribute>
+            <Attribute default="Breach Title" check="String" primary="False">Breach Title</Attribute>
+            <Attribute default="Breach Domain" check="String" primary="False">Breach Domain</Attribute>
+            <Attribute default="0" check="Numbers" primary="False">Breach Pwn Count</Attribute>
+            <Attribute default="Breach Description" check="String" primary="False">Breach Description</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Sensitive</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Verified</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Fabricated</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Retired</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Spam List</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Malware</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Breach Added Date</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Breach Modified Date</Attribute>
+        </Attributes>
+        <Icon>
+            Default.svg
+        </Icon>
+    </Data_Breach>
+    <Paste_Data_Leak>
+        <Attributes>
+            <Attribute default="Paste Identifier" check="String" primary="True">Paste Identifier</Attribute>
+            <Attribute default="Paste Title" check="String" primary="False">Paste Title</Attribute>
+            <Attribute default="Paste Source" check="String" primary="False">Paste Source</Attribute>
+            <Attribute default="Paste ID" check="String" primary="False">Paste ID</Attribute>
+            <Attribute default="0" check="Numbers" primary="False">Paste Email Count</Attribute>
+        </Attributes>
+        <Icon>
+            Default.svg
+        </Icon>
+    </Paste_Data_Leak>
+</HIBP>

Modules/BreachData/HIBPBreachToDomain.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+class HIBPBreachToDomain:
+
+    name = "HIBP Breach To Domain"
+    category = "Leaked Data"
+    description = "Get the domain of the primary website that a data breach occurred on."
+    originTypes = {'Data Breach'}
+    resultTypes = {'Domain'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            domainMaybe = entity.get('Breach Domain')
+            if isinstance(domainMaybe, str):
+                if domainMaybe.strip() != '':
+                    returnResults.append([{'Domain Name': domainMaybe,
+                                           'Entity Type': 'Domain'},
+                                          {entity['uid']: {'Resolution': 'Data Breach to Domain',
+                                                           'Notes': ''}}])
+
+        return returnResults

Modules/BreachData/HaveIBeenPwnedBreachDomains.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedBreachDomains:
+    name = "HIBP Breach Domain Lookup"
+    category = "Leaked Data"
+    description = "Find breaches associated with a specified domain."
+    originTypes = {'Domain'}
+    resultTypes = {'Data Breach'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+
+        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
+        from PySide6.QtGui import QImage
+
+        baseURL = "https://haveibeenpwned.com/api/v3/breaches?domain="
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            primaryField = entity['Domain Name']
+            breachInfoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            elif statusCode == 200:
+                breachContent = json.loads(breachInfoRequest.content)
+
+                for breach in breachContent:
+                    try:
+                        breachLogoIconRequest = requests.get(breach['LogoPath'])
+                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
+                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
+                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
+
+                        # Rotate the breach domain logo upside down
+                        breachIconImageRotated = breachIconImageScaled.mirrored()
+
+                        breachIconByteArrayFin = QByteArray()
+                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
+                        breachImageBuffer.open(QIODevice.WriteOnly)
+                        breachIconImageRotated.save(breachImageBuffer, "PNG")
+                        breachImageBuffer.close()
+                    except Exception:
+                        breachIconByteArrayFin = None
+
+                    # If Breach Date is None, then default to entity creation date.
+                    returnResults.append([{'Breach Name': breach['Name'],
+                                           'Breach Title': breach['Title'],
+                                           'Breach Domain': breach['Domain'],
+                                           'Breach Pwn Count': str(breach['PwnCount']),
+                                           'Breach Description': breach['Description'],
+                                           'Breach Is Sensitive': str(breach['IsSensitive']),
+                                           'Breach Is Verified': str(breach['IsVerified']),
+                                           'Breach Is Fabricated': str(breach['IsFabricated']),
+                                           'Breach Is Retired': str(breach['IsRetired']),
+                                           'Breach Is Spam List': str(breach['IsSpamList']),
+                                           'Breach Is Malware': str(breach['IsMalware']),
+                                           'Breach Added Date': breach['AddedDate'],
+                                           'Breach Modified Date': breach['ModifiedDate'],
+                                           'Entity Type': 'Data Breach',
+                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
+                                           'Date Created': breach['BreachDate']},
+                                          {entity['uid']: {'Resolution': 'Contained in Breach',
+                                                           'Notes': ''}}])
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedBreaches.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedBreaches:
+    name = "HIBP Breach Lookup"
+    category = "Leaked Data"
+    description = "Find all breaches that an account has been involved in. Note that Date Created for breaches is an " \
+                  "estimate."
+    originTypes = {'Email Address', 'Phone Number'}
+    resultTypes = {'Data Breach'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+        from urllib.parse import quote_plus
+
+        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
+        from PySide6.QtGui import QImage
+
+        baseURL = "https://haveibeenpwned.com/api/v3/breachedaccount/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            primaryField = entity[list(entity)[1]]
+            breachInfoRequest = requests.get(baseURL + quote_plus(primaryField) + '?truncateResponse=false',
+                                             headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            elif statusCode == 200:
+                breachContent = json.loads(breachInfoRequest.content)
+
+                for breach in breachContent:
+                    try:
+                        breachLogoIconRequest = requests.get(breach['LogoPath'])
+                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
+                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
+                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
+
+                        # Rotate the breach domain logo upside down
+                        breachIconImageRotated = breachIconImageScaled.mirrored()
+
+                        breachIconByteArrayFin = QByteArray()
+                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
+                        breachImageBuffer.open(QIODevice.WriteOnly)
+                        breachIconImageRotated.save(breachImageBuffer, "PNG")
+                        breachImageBuffer.close()
+                    except Exception:
+                        breachIconByteArrayFin = None
+
+                    # If Breach Date is None, then default to entity creation date.
+                    returnResults.append([{'Breach Name': breach['Name'],
+                                           'Breach Title': breach['Title'],
+                                           'Breach Domain': breach['Domain'],
+                                           'Breach Pwn Count': str(breach['PwnCount']),
+                                           'Breach Description': breach['Description'],
+                                           'Breach Is Sensitive': str(breach['IsSensitive']),
+                                           'Breach Is Verified': str(breach['IsVerified']),
+                                           'Breach Is Fabricated': str(breach['IsFabricated']),
+                                           'Breach Is Retired': str(breach['IsRetired']),
+                                           'Breach Is Spam List': str(breach['IsSpamList']),
+                                           'Breach Is Malware': str(breach['IsMalware']),
+                                           'Breach Added Date': breach['AddedDate'],
+                                           'Breach Modified Date': breach['ModifiedDate'],
+                                           'Entity Type': 'Data Breach',
+                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
+                                           'Date Created': breach['BreachDate']},
+                                          {entity['uid']: {'Resolution': 'Contained in Breach',
+                                                           'Notes': ''}}])
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPassword.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPassword:
+    name = "HIBP Password Lookup"
+    category = "Leaked Data"
+    description = "Check whether the given password was found in breaches."
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from time import sleep
+        from hashlib import sha1
+
+        baseURL = "https://api.pwnedpasswords.com/range/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+
+            primaryField = sha1(entity[list(entity)[1]].encode('utf-8')).hexdigest().upper()
+            hashPrefix = primaryField[:5]
+            hashSuffix = primaryField[5:]
+
+            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            elif statusCode == 200:
+                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
+
+                for password in pwnedPasswordContent:
+                    if hashSuffix in password:
+                        returnResults.append([{'Phrase': 'Password Hash Found ' + password.split(':')[1] +
+                                                         ' times in breach data.',
+                                               'Entity Type': 'Phrase'},
+                                              {entity['uid']: {'Resolution': 'Pwned Password',
+                                                               'Notes': ''}}])
+                        break
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPasswordHash.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPasswordHash:
+    name = "HIBP Password Hash Lookup"
+    category = "Leaked Data"
+    description = "Check whether the given password hash was found in breaches."
+    originTypes = {'Hash', 'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from time import sleep
+
+        baseURL = "https://api.pwnedpasswords.com/range/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+
+            primaryField = entity[list(entity)[1]].upper()
+            if len(primaryField) != 40:
+                continue
+            hashPrefix = primaryField[:5]
+            hashSuffix = primaryField[5:]
+
+            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            elif statusCode == 200:
+                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
+
+                for password in pwnedPasswordContent:
+                    if hashSuffix in password:
+                        returnResults.append([{'Phrase': 'Password Hash Found ' + password.split(':')[1] +
+                                                         ' times in breach data.',
+                                               'Entity Type': 'Phrase'},
+                                              {entity['uid']: {'Resolution': 'Pwned Password',
+                                                               'Notes': ''}}])
+                        break
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPastes.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPastes:
+    name = "HIBP Paste Lookup"
+    category = "Leaked Data"
+    description = "Find all pastes that an account has been involved in."
+    originTypes = {'Email Address'}
+    resultTypes = {'Paste Data Leak'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+        from urllib.parse import quote_plus
+
+        baseURL = "https://haveibeenpwned.com/api/v3/pasteaccount/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            emailAddress = entity['Email Address']
+            pasteInfoRequest = requests.get(baseURL + quote_plus(emailAddress), headers=requestHeaders)
+            statusCode = pasteInfoRequest.status_code
+            if statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            elif statusCode == 200:
+                pasteContent = json.loads(pasteInfoRequest.content)
+
+                for paste in pasteContent:
+                    pasteID = paste['Id']
+                    pasteSource = paste['Source']
+
+                    # If Paste Date is None, then default to entity creation date.
+                    returnResults.append([{'Paste Identifier': pasteSource + ' | ' + pasteID,
+                                           'Paste Title': paste['Title'],
+                                           'Paste Source': pasteSource,
+                                           'Paste ID': pasteID,
+                                           'Paste Email Count': str(paste['EmailCount']),
+                                           'Entity Type': 'Paste Data Leak',
+                                           'Date Created': paste['Date']},
+                                          {entity['uid']: {'Resolution': 'Contained in Paste',
+                                                           'Notes': ''}}])
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/CryptoCurrency/BlockChainAPI.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainAPI:
-    # A string that is treated as the name of this resolution.
-    name = "BlockChainAPI Get Transaction Details"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes of balance, sent, received info"
-
-    originTypes = {'Crypto Wallet', 'Phrase'}
-
-    resultTypes = {'Currency'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import datetime
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            search_address = entity[list(entity)[1]]
-
-            try:
-                getSentByAddress = requests.get(
-                    f'https://blockchain.info/q/getsentbyaddress/{search_address}?confirmations=6')
-                if getSentByAddress.status_code == 404:
-                    continue
-                time.sleep(1)
-                getReceivedByAddress = requests.get(
-                    f'https://blockchain.info/q/getreceivedbyaddress/{search_address}?confirmations=6')
-                time.sleep(1)
-                addressBalance = requests.get(
-                    f'https://blockchain.info/q/addressbalance/{search_address}?confirmations=6')
-                time.sleep(1)
-                addressFirstSeen = requests.get(
-                    f'https://blockchain.info/q/addressfirstseen/{search_address}?confirmations=6')
-                time.sleep(1)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            dateCreated = str(datetime.datetime.fromtimestamp(float(addressFirstSeen.text))).replace(" ", 'T')
-
-            returnResults.append(
-                [{'Amount': str(int(getSentByAddress.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'sent by address',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'Sent by address',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Amount': str(int(getReceivedByAddress.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'received by address',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'Received by address',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Amount': str(int(addressBalance.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'address balance',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'Address balance',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Date': str(dateCreated),
-                  'Notes': 'first seen date',
-                  'Entity Type': 'Date'},
-                 {uid: {'Resolution': 'First seen date',
-                        'Notes': ''}}])
-            return returnResults

Modules/CryptoCurrency/BlockChainAddressDestinations.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+
+class BlockChainAddressDestinations:
+
+    name = "Get Outbound Transactions for Bitcoin Address"
+    category = "CryptoCurrency"
+    description = "Returns the Bitcoin transactions where cryptocurrency was sent from this address."
+    originTypes = {'BTC Address'}
+    resultTypes = {'BTC Transaction'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+
+        returnResults = []
+
+        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity['BTC Address']
+
+            try:
+                addressDetails = requests.get(apiEndpointAddress + primaryField).json()
+                if addressDetails.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            blockTransactions = addressDetails.get('txs', [])
+
+            for transaction in blockTransactions:
+                inputValue = 0
+                isInAddr = False
+                for transactionInput in transaction.get('inputs', []):
+                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
+                    if transactionInput['prev_out']['addr'] == primaryField:
+                        isInAddr = True
+
+                if not isInAddr:
+                    continue
+
+                outputValue = 0
+                for transactionOutput in transaction.get('out', []):
+                    outputValue += (int(transactionOutput['value']) / 100000000)
+
+                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
+
+                returnResults.append(
+                    [{'Transaction Hash': transaction['hash'],
+                      'Input Value (BTC)': str(inputValue),
+                      'Output Value (BTC)': str(outputValue),
+                      'Fee': str(transaction['fee']),
+                      'Number of Inputs': str(transaction['vin_sz']),
+                      'Number of Outputs': str(transaction['vout_sz']),
+                      'Transaction Index': str(transaction['tx_index']),
+                      'Size': str(transaction['size']),
+                      'Height': str(transaction['block_height']),
+                      'Entity Type': 'BTC Transaction',
+                      'Date Created': timestamp},
+                     {uid: {'Resolution': 'BTC Transaction',
+                            'Notes': ''}}])
+
+            time.sleep(5)
+
+        return returnResults

Modules/CryptoCurrency/BlockChainAddressSources.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+
+class BlockChainAddressSources:
+
+    name = "Get Inbound Transactions for Bitcoin Address"
+    category = "CryptoCurrency"
+    description = "Returns the Bitcoin transactions where cryptocurrency was sent to this address."
+    originTypes = {'BTC Address'}
+    resultTypes = {'BTC Transaction'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+
+        returnResults = []
+        returnResultResolutions = {}
+
+        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
+
+        for entity in entityJsonList:
+            primaryField = entity['BTC Address']
+
+            try:
+                addressDetails = requests.get(apiEndpointAddress + primaryField).json()
+                if addressDetails.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            blockTransactions = addressDetails.get('txs', [])
+
+            for transaction in blockTransactions:
+                outputValue = 0
+                isOutAddr = False
+                for transactionOutput in transaction.get('out', []):
+                    outputValue += (int(transactionOutput['value']) / 100000000)
+                    if transactionOutput['addr'] == primaryField:
+                        isOutAddr = True
+
+                if not isOutAddr:
+                    continue
+
+                inputValue = 0
+                for transactionInput in transaction.get('inputs', []):
+                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
+
+                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
+                returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
+
+                returnResults.append(
+                    [{'Transaction Hash': transaction['hash'],
+                      'Input Value (BTC)': str(inputValue),
+                      'Output Value (BTC)': str(outputValue),
+                      'Fee': str(transaction['fee']),
+                      'Number of Inputs': str(transaction['vin_sz']),
+                      'Number of Outputs': str(transaction['vout_sz']),
+                      'Transaction Index': str(transaction['tx_index']),
+                      'Size': str(transaction['size']),
+                      'Height': str(transaction['block_height']),
+                      'Entity Type': 'BTC Transaction',
+                      'Date Created': timestamp},
+                     {'^^^': {'Resolution': 'NULL',
+                              'Notes': ''}}])
+
+            time.sleep(5)
+            returnResults.append([{'BTC Address': primaryField,
+                                   'Entity Type': 'BTC Address'},
+                                  returnResultResolutions])
+            returnResultResolutions = {}
+        return returnResults

Modules/CryptoCurrency/BlockChainBlock.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+
+class BlockChainBlock:
+
+    name = "Get Bitcoin Block Details"
+    category = "CryptoCurrency"
+    description = "Returns the details of a particular bitcoin block."
+    originTypes = {'Hash', 'Phrase', 'BTC Block'}
+    resultTypes = {'BTC Block'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/rawblock/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity[list(entity)[1]]
+
+            try:
+                details = requests.get(apiEndpoint + primaryField).json()
+                if details.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
+
+            returnResults.append(
+                [{'Block Address': details['hash'],
+                  'Previous Block': details['prev_block'],
+                  'Merkle Root': details['mrkl_root'],
+                  'Relayed By': details['relayed_by'],
+                  'Nonce': str(details['nonce']),
+                  'Bits': str(details['bits']),
+                  'Size': str(details['size']),
+                  'Block Index': str(details['block_index']),
+                  'Height': str(details['height']),
+                  'Main Chain': str(details['main_chain']),
+                  'Entity Type': 'BTC Block',
+                  'Date Created': timestamp},
+                 {uid: {'Resolution': 'Bitcoin Block Details',
+                        'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainBlockHeight.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+
+class BlockChainBlockHeight:
+
+    name = "Get Bitcoin Blocks At Height"
+    category = "CryptoCurrency"
+    description = "Returns the details of all bitcoin blocks at the specified height."
+    originTypes = {'Phrase'}
+    resultTypes = {'BTC Block'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/block-height/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            try:
+                primaryField = int(entity['Phrase'])
+            except ValueError:
+                continue
+
+            try:
+                heightDetails = requests.get(apiEndpoint + str(primaryField)).json()
+                if heightDetails.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            for details in heightDetails['blocks']:
+                timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
+
+                returnResults.append(
+                    [{'Block Address': details['hash'],
+                      'Previous Block': details['prev_block'],
+                      'Merkle Root': details['mrkl_root'],
+                      'Relayed By': details['relayed_by'],
+                      'Nonce': str(details['nonce']),
+                      'Bits': str(details['bits']),
+                      'Size': str(details['size']),
+                      'Block Index': str(details['block_index']),
+                      'Height': str(details['height']),
+                      'Main Chain': str(details['main_chain']),
+                      'Entity Type': 'BTC Block',
+                      'Date Created': timestamp},
+                     {uid: {'Resolution': 'Bitcoin Block Address',
+                            'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainBlockNext.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+
+class BlockChainBlockNext:
+
+    name = "Get Next Bitcoin Block"
+    category = "CryptoCurrency"
+    description = "Returns the details of the next bitcoin block."
+    originTypes = {'BTC Block'}
+    resultTypes = {'BTC Block'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/rawblock/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity['Block Address']
+
+            try:
+                currDetails = requests.get(apiEndpoint + primaryField).json()
+                if currDetails.get('error') is not None:
+                    continue
+                nextBlockHashList = currDetails.get('next_block')
+                # Ignore nonexistent or indeterminate 'next' blocks.
+                if nextBlockHashList is None or len(nextBlockHashList) > 1:
+                    continue
+                time.sleep(5)
+                details = requests.get(apiEndpoint + nextBlockHashList[0]).json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
+
+            returnResults.append(
+                [{'Block Address': details['hash'],
+                  'Previous Block': details['prev_block'],
+                  'Merkle Root': details['mrkl_root'],
+                  'Relayed By': details['relayed_by'],
+                  'Nonce': str(details['nonce']),
+                  'Bits': str(details['bits']),
+                  'Size': str(details['size']),
+                  'Block Index': str(details['block_index']),
+                  'Height': str(details['height']),
+                  'Main Chain': str(details['main_chain']),
+                  'Entity Type': 'BTC Block',
+                  'Date Created': timestamp},
+                 {uid: {'Resolution': 'Next BTC Block',
+                        'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainBlockPrev.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+
+class BlockChainBlockPrev:
+
+    name = "Get Previous Bitcoin Block"
+    category = "CryptoCurrency"
+    description = "Returns the details of the previous bitcoin block."
+    originTypes = {'BTC Block'}
+    resultTypes = {'BTC Block'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/rawblock/'
+
+        for entity in entityJsonList:
+            primaryField = entity['Block Address']
+
+            try:
+                currDetails = requests.get(apiEndpoint + primaryField).json()
+                if currDetails.get('error') is not None:
+                    continue
+                prevBlockHash = currDetails.get('prev_block')
+                # Ignore first block.
+                if prevBlockHash == "0000000000000000000000000000000000000000000000000000000000000000":
+                    continue
+                time.sleep(5)
+                details = requests.get(apiEndpoint + prevBlockHash).json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
+
+            returnResults.append(
+                [{'Block Address': details['hash'],
+                  'Previous Block': details['prev_block'],
+                  'Merkle Root': details['mrkl_root'],
+                  'Relayed By': details['relayed_by'],
+                  'Nonce': str(details['nonce']),
+                  'Bits': str(details['bits']),
+                  'Size': str(details['size']),
+                  'Block Index': str(details['block_index']),
+                  'Height': str(details['height']),
+                  'Main Chain': str(details['main_chain']),
+                  'Entity Type': 'BTC Block',
+                  'Date Created': timestamp},
+                 {'^^^': {'Resolution': 'NULL'}}])
+
+            returnResults.append([{'Block Address': primaryField,
+                                   'Entity Type': 'BTC Block'},
+                                  {len(returnResults) - 1: {'Resolution': 'Next BTC Block'}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainBlockTransactions.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+
+class BlockChainBlockTransactions:
+
+    name = "Get Bitcoin Block Transactions"
+    category = "CryptoCurrency"
+    description = "Returns the transactions that happened in a particular bitcoin block."
+    originTypes = {'BTC Block'}
+    resultTypes = {'BTC Transaction'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/rawblock/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity['Block Address']
+
+            try:
+                details = requests.get(apiEndpoint + primaryField).json()
+                if details.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            blockTransactions = details.get('tx', [])
+
+            for transaction in blockTransactions:
+                inputValue = 0
+                for transactionInput in transaction.get('inputs', []):
+                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
+
+                outputValue = 0
+                for transactionOutput in transaction.get('out', []):
+                    outputValue += (int(transactionOutput['value']) / 100000000)
+
+                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
+
+                returnResults.append(
+                    [{'Transaction Hash': transaction['hash'],
+                      'Input Value (BTC)': str(inputValue),
+                      'Output Value (BTC)': str(outputValue),
+                      'Fee': str(transaction['fee']),
+                      'Number of Inputs': str(transaction['vin_sz']),
+                      'Number of Outputs': str(transaction['vout_sz']),
+                      'Transaction Index': str(transaction['tx_index']),
+                      'Size': str(transaction['size']),
+                      'Height': str(transaction['block_height']),
+                      'Entity Type': 'BTC Transaction',
+                      'Date Created': timestamp},
+                     {uid: {'Resolution': 'Bitcoin Block Address',
+                            'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainTransaction.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+
+class BlockChainTransaction:
+
+    name = "Get Bitcoin Transaction"
+    category = "CryptoCurrency"
+    description = "Returns the details of the specified Bitcoin transaction."
+    originTypes = {'BTC Transaction', 'Hash', 'Phrase'}
+    resultTypes = {'BTC Transaction'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        from datetime import datetime
+        returnResults = []
+
+        apiEndpoint = 'https://blockchain.info/rawtx/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity[list(entity)[1]]
+
+            try:
+                transaction = requests.get(apiEndpoint + primaryField).json()
+                if transaction.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
+
+            inputValue = 0
+            for transactionInput in transaction.get('inputs', []):
+                inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
+
+            outputValue = 0
+            for transactionOutput in transaction.get('out', []):
+                outputValue += (int(transactionOutput['value']) / 100000000)
+
+            returnResults.append(
+                [{'Transaction Hash': transaction['hash'],
+                  'Input Value (BTC)': str(inputValue),
+                  'Output Value (BTC)': str(outputValue),
+                  'Fee': str(transaction['fee']),
+                  'Number of Inputs': str(transaction['vin_sz']),
+                  'Number of Outputs': str(transaction['vout_sz']),
+                  'Transaction Index': str(transaction['tx_index']),
+                  'Size': str(transaction['size']),
+                  'Height': str(transaction['block_height']),
+                  'Entity Type': 'BTC Transaction',
+                  'Date Created': timestamp},
+                 {uid: {'Resolution': 'Bitcoin Transaction Information',
+                        'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainTransactionDestinations.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+
+class BlockChainTransactionDestinations:
+
+    name = "Get Bitcoin Transaction Destinations"
+    category = "CryptoCurrency"
+    description = "Returns the Bitcoin addresses that received cryptocurrency in the specified transaction."
+    originTypes = {'BTC Transaction'}
+    resultTypes = {'BTC Address'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        returnResults = []
+
+        apiEndpointTransaction = 'https://blockchain.info/rawtx/'
+        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primaryField = entity['Transaction Hash']
+
+            try:
+                transaction = requests.get(apiEndpointTransaction + primaryField).json()
+                if transaction.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            for transactionInput in transaction.get('out', []):
+                time.sleep(5)
+                inputAddress = transactionInput['addr']
+                try:
+                    details = requests.get(apiEndpointAddress + inputAddress).json()
+                    if details.get('error') is not None:
+                        continue
+                except requests.exceptions.ConnectionError:
+                    return "Please check your internet connection"
+
+                returnResults.append(
+                    [{'BTC Address': details['address'],
+                      'Total Transactions': str(details['n_tx']),
+                      'Unredeemed Transactions': str(details['n_unredeemed']),
+                      'Total BTC Received': str(details['total_received'] / 100000000),
+                      'Total BTC Sent': str(details['total_sent'] / 100000000),
+                      'Current Balance': str(details['final_balance'] / 100000000),
+                      'Entity Type': 'BTC Address'},
+                     {uid: {'Resolution': 'Bitcoin Transaction',
+                            'Notes': ''}}])
+
+            time.sleep(5)
+        return returnResults

Modules/CryptoCurrency/BlockChainTransactionSources.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+
+class BlockChainTransactionSources:
+
+    name = "Get Bitcoin Transaction Sources"
+    category = "CryptoCurrency"
+    description = "Returns the Bitcoin addresses that sent cryptocurrency in the specified transaction."
+    originTypes = {'BTC Transaction'}
+    resultTypes = {'BTC Address'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        returnResults = []
+        returnResultResolutions = {}
+
+        apiEndpointTransaction = 'https://blockchain.info/rawtx/'
+        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
+
+        for entity in entityJsonList:
+            primaryField = entity['Transaction Hash']
+
+            try:
+                transaction = requests.get(apiEndpointTransaction + primaryField).json()
+                if transaction.get('error') is not None:
+                    continue
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            for transactionInput in transaction.get('inputs', []):
+                time.sleep(5)
+                inputAddress = transactionInput['prev_out']['addr']
+                try:
+                    details = requests.get(apiEndpointAddress + inputAddress).json()
+                    if details.get('error') is not None:
+                        continue
+                except requests.exceptions.ConnectionError:
+                    return "Please check your internet connection"
+
+                returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
+                returnResults.append(
+                    [{'BTC Address': details['address'],
+                      'Total Transactions': str(details['n_tx']),
+                      'Unredeemed Transactions': str(details['n_unredeemed']),
+                      'Total BTC Received': str(details['total_received'] / 100000000),
+                      'Total BTC Sent': str(details['total_sent'] / 100000000),
+                      'Current Balance': str(details['final_balance'] / 100000000),
+                      'Entity Type': 'BTC Address'},
+                     {'^^^': {'Resolution': 'NULL',
+                              'Notes': ''}}])
+
+            time.sleep(5)
+            # Re-add the source entity so that we can point to it.
+            # Only include the primary field, in case the rest of the fields were updated in the meantime.
+            returnResults.append([{'Transaction Hash': primaryField,
+                                   'Entity Type': 'BTC Transaction'},
+                                  returnResultResolutions])
+            returnResultResolutions = {}
+        return returnResults

Modules/CryptoCurrency/CryptoCurrency.xml

@@ -1,10 +1,26 @@
 <CryptoCurrency>
-  <Crypto_Wallet>
-    <Attributes>
-      <Attribute default="Wallet Address" check="String" primary="True">Wallet Address</Attribute>
-    </Attributes>
-    <Icon>
-      Default.svg
-    </Icon>
-  </Crypto_Wallet>
+    <BTC_Block>
+        <Attributes>
+          <Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Block Address</Attribute>
+        </Attributes>
+        <Icon>
+          Bitcoin.svg
+        </Icon>
+    </BTC_Block>
+    <BTC_Transaction>
+        <Attributes>
+            <Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Transaction Hash</Attribute>
+        </Attributes>
+        <Icon>
+            Transaction.svg
+        </Icon>
+    </BTC_Transaction>
+    <BTC_Address>
+        <Attributes>
+            <Attribute default="1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" check="String" primary="True">BTC Address</Attribute>
+        </Attributes>
+        <Icon>
+            CryptoWallet.svg
+        </Icon>
+    </BTC_Address>
 </CryptoCurrency>

Modules/CryptoCurrency/EtherScanGetBalance.py

@@ -3,27 +3,29 @@
 
 class EtherScanGetBalance:
     name = "EtherScan.io Get Balance"
+    category = "CryptoCurrency"
     description = "EtherScan get the balance of the selected account"
     originTypes = {"Crypto Wallet"}
     resultTypes = {'Crypto Wallet'}
     parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io",
+                                                       "https://etherscan.io.",
                                         'type': 'String',
                                         'value': '',
                                         'global': True}}
 
     def resolution(self, entityJsonList, parameters):
         import requests
+        import time
 
         return_result = []
 
         api_key = parameters['EtherScan API Key']
-        url = "https://api.etherscan.io/api"
 
         for entity in entityJsonList:
             uid = entity['uid']
-            primary_field = entity[list(entity)[1]]
-            crafted_url = f"{url}?module=account&action=balance&address={primary_field}&tag=latest&apikey={api_key}"
+            primary_field = entity['Wallet Address']
+            crafted_url = f"https://api.etherscan.io/api?module=account&action=balance" \
+                          f"&address={primary_field}&tag=latest&apikey={api_key}"
             try:
                 response = requests.get(crafted_url)
             except requests.exceptions.ConnectionError:
@@ -33,4 +35,6 @@ class EtherScanGetBalance:
                                    'Currency': 'Ethereum',
                                    'Entity Type': 'Currency'},
                                   {uid: {'Resolution': 'EtherScan.io Account Balance', 'Notes': ''}}])
+            time.sleep(0.2)
+
         return return_result

Modules/CryptoCurrency/EtherScanGetBlocksMined.py

@@ -3,28 +3,29 @@
 
 class EtherScanGetBlocksMined:
     name = "EtherScan.io Get Blocks Mined"
+    category = "CryptoCurrency"
     description = "EtherScan Blocks Mined from the selected account"
     originTypes = {"Crypto Wallet"}
     resultTypes = {'Crypto Wallet'}
     parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io",
+                                                       "https://etherscan.io.",
                                         'type': 'String',
                                         'value': '',
                   'global': True}}
 
     def resolution(self, entityJsonList, parameters):
         import requests
+        import time
 
         return_result = []
 
         api_key = parameters['EtherScan API Key']
-        url = "https://api.etherscan.io/api"
 
         for entity in entityJsonList:
             uid = entity['uid']
-            primary_field = entity[list(entity)[1]]
-            crafted_url =\
-                f"{url}?module=account&action=getminedblocks&address={primary_field}&tag=latest&apikey={api_key}"
+            primary_field = entity['Wallet Address']
+            crafted_url = f"https://api.etherscan.io/api?module=account&action=getminedblocks" \
+                          f"&address={primary_field}&tag=latest&apikey={api_key}"
             try:
                 response = requests.get(crafted_url)
             except requests.exceptions.ConnectionError:
@@ -33,4 +34,6 @@ class EtherScanGetBlocksMined:
             return_result.append([{'Phrase': response['result'],
                                    'Entity Type': 'Phrase'},
                                   {uid: {'Resolution': 'EtherScan.io Blocks Mined', 'Notes': ''}}])
+            time.sleep(0.2)
+
         return return_result

Modules/CryptoCurrency/ToCryptoWallet.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+
+class ToCryptoWallet:
+    name = "BTC Address To Crypto Wallet"
+    category = "CryptoCurrency"
+    description = "Convert BTC Address entities to Crypto Wallet entities."
+    originTypes = {'BTC Address'}
+    resultTypes = {'Crypto Wallet'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['BTC Address']
+            returnResults.append([{'Wallet Address': primaryField,
+                                   'Currency Name': 'Bitcoin',
+                                   'Entity Type': 'Crypto Wallet'},
+                                  {entity['uid']: {'Resolution': 'To Crypto Wallet',
+                                                   'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/CompanyInfo.py

@@ -4,6 +4,8 @@ class CompanyInfo:
     # A string that is treated as the name of this resolution.
     name = "Get Company Info"
 
+    category = "EDGAR Info"
+
     # A string that describes this resolution.
     description = "Returns Nodes containing Company Information"
 

Modules/EDGAR_&_EFD/CompanyToCIK.py

@@ -5,6 +5,8 @@ class CompanyToCIK:
     # A string that is treated as the name of this resolution.
     name = "Get CIK ID From Company"
 
+    category = "EDGAR Info"
+
     # A string that describes this resolution.
     description = "Returns Nodes of contact info for websites"
 
@@ -17,7 +19,7 @@ class CompanyToCIK:
     def resolution(self, entityJsonList, parameters):
         import re
         from bs4 import BeautifulSoup
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
 
         returnResults = []
         index_of_child = []
@@ -43,6 +45,8 @@ class CompanyToCIK:
                         break
                     except TimeoutError:
                         pass
+                    except Error:
+                        break
                 if not pageResolved:
                     continue
 

Modules/EDGAR_&_EFD/EFDByFromDate.py

@@ -3,6 +3,7 @@
 
 class EFDByFromDate:
     name = 'Get EFD Reports From Date'
+    category = "US Senate Financial Info"
     description = 'Get EFD reports starting from the date specified by the input entities.'
     originTypes = {'Date'}
     resultTypes = {'Politically Exposed Person', 'Website'}
@@ -33,7 +34,7 @@ class EFDByFromDate:
 
     def resolution(self, entityJsonList, parameters):
         from datetime import datetime
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
 
         returnResults = []
@@ -69,13 +70,17 @@ class EFDByFromDate:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return "Could not access efdsearch website."
+                return "Could not access EFD Search website."
 
             try:
                 page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
             except TimeoutError:
-                return "The efdsearch website is unresponsive"
+                return "The EFD search website is unresponsive."
+            except Error:
+                return "Connection Error."
             page.wait_for_timeout(1000)
 
             for entity in entityJsonList:
@@ -99,6 +104,8 @@ class EFDByFromDate:
                         break
                     except TimeoutError:
                         pass
+                    except Error:
+                        break
                 if not pageResolved:
                     continue
 

Modules/EDGAR_&_EFD/EFDByName.py

@@ -3,6 +3,7 @@
 
 class EFDByName:
     name = 'Get EFD Reports by Name'
+    category = "US Senate Financial Info"
     description = 'Get EFD reports concerning the people specified by the input entities.'
     originTypes = {'Person', 'Politically Exposed Person'}
     resultTypes = {'Website'}

Modules/EDGAR_&_EFD/EFDByToDate.py

@@ -3,6 +3,7 @@
 
 class EFDByToDate:
     name = 'Get EFD Reports To Date'
+    category = "US Senate Financial Info"
     description = 'Get EFD reports ending at the date specified by the input entities.'
     originTypes = {'Date'}
     resultTypes = {'Politically Exposed Person', 'Website'}
@@ -33,7 +34,7 @@ class EFDByToDate:
 
     def resolution(self, entityJsonList, parameters):
         from datetime import datetime
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
 
         returnResults = []
@@ -69,13 +70,17 @@ class EFDByToDate:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return "Could not access efdsearch website."
+                return "Could not access EFD Search website."
 
             try:
                 page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
             except TimeoutError:
-                return "The efdsearch website is unresponsive"
+                return "The EFD search website is unresponsive."
+            except Error:
+                return "Connection Error."
             page.wait_for_timeout(1000)
 
             for entity in entityJsonList:
@@ -99,6 +104,8 @@ class EFDByToDate:
                         break
                     except TimeoutError:
                         pass
+                    except Error:
+                        break
                 if not pageResolved:
                     continue
 

Modules/EDGAR_&_EFD/FramesLookUp.py

@@ -4,6 +4,8 @@ class FramesLookUp:
     # A string that is treated as the name of this resolution.
     name = "Frames Look Up"
 
+    category = "EDGAR Info"
+
     # A string that describes this resolution.
     description = "Returns Frame Forms"
 

Modules/EDGAR_&_EFD/Get10KForms.py

@@ -4,6 +4,8 @@ class Get10KForms:
     # A string that is treated as the name of this resolution.
     name = "Get Recent 10-K Forms"
 
+    category = "EDGAR Info"
+
     # A string that describes this resolution.
     description = "Returns Nodes 10-K Forms"