Push commits:

Move saved Resolutions parameters to global scope.

Move all Program settings to global scope.

Update & Refactor Settings Object.

Update Whats My Name - make detection more precise & add sites.

Update README.md

Update version number

Finalize importing tabs from TOR.

Change settings labels so they look better in menus. Add Tor browser path setting.

Fix bug with deselected entities when zooming out.

Transition to pyqtdarktheme for theme & styling.

Check if firefox snap exists if firefox package is not present.

Make sure that every file from the directories holding resolutions is present.

Refactor importing of tabs.

.gitignore

@@ -0,0 +1,6 @@
+/magic/COPYING.file
+/magic/COPYING.libgnurx
+/magic/file.exe
+/magic/libgnurx-0.dll
+/magic/libmagic-1.dll
+/magic/magic.mgc

CONTRIBUTING

@@ -0,0 +1,3 @@
+By contributing to this software, the contributor hereby grants to AccentuSoft and to recipients of software distributed by AccentuSoft a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer all or parts of this software, where such license applies only to those patent claims licensable by the contributor that are necessarily infringed by the contributor's contribution(s) alone or by combination of the contributor's contribution(s) with the all or parts of this software to which such contribution(s) was submitted.
+
+In non-legalese, this means that any contributions made to this software can be used by AccentuSoft and any other user for any purpose without concern that the original author will revoke permission to use their contribution, or take anyone to court. Make sure that all your contributions consist of things that you want to contribute to the project. In all cases, the project's license still applies.

Core/Entities/Devices.xml

@@ -0,0 +1,29 @@
+<Devices>
+    <Mobile_Phone>
+        <Attributes>
+            <Attribute default="a:b:c:d:e:f" check="String" primary="True">Device UID</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
+        </Attributes>
+        <Icon>
+            Mobile_Device.svg
+        </Icon>
+    </Mobile_Phone>
+    <Computer>
+        <Attributes>
+            <Attribute default="User Workstation" check="String" primary="True">Hostname</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
+        </Attributes>
+        <Icon>
+            Computer.svg
+        </Icon>
+    </Computer>
+    <Camera>
+        <Attributes>
+            <Attribute default="a:b:c:d:e:f" check="String" primary="True">Device UID</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
+        </Attributes>
+        <Icon>
+            Camera.svg
+        </Icon>
+    </Camera>
+</Devices>

Core/Entities/Financials.xml

@@ -0,0 +1,46 @@
+<Financials>
+    <Bank_Account>
+        <Attributes>
+            <Attribute default="Account Number" check="String" primary="True">Account Number</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Bank Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Branch Number</Attribute>
+        </Attributes>
+        <Icon>
+            Bank_Account.svg
+        </Icon>
+    </Bank_Account>
+    <Currency>
+        <Attributes>
+            <Attribute default="Money Amount" check="String" primary="True">Amount</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Currency Type</Attribute>
+        </Attributes>
+        <Icon>
+            Currency.svg
+        </Icon>
+    </Currency>
+    <Ticker>
+        <Attributes>
+            <Attribute default="Ticker ID" check="String" primary="True">Ticker ID</Attribute>
+        </Attributes>
+        <Icon>
+            Share.svg
+        </Icon>
+    </Ticker>
+    <Portfolio>
+        <Attributes>
+            <Attribute default="Portfolio Name" check="String" primary="True">Label</Attribute>
+        </Attributes>
+        <Icon>
+            Portfolio.svg
+        </Icon>
+    </Portfolio>
+    <Crypto_Wallet>
+        <Attributes>
+          <Attribute default="Wallet Address" check="String" primary="True">Wallet Address</Attribute>
+          <Attribute default="Bitcoin" check="String" primary="False">Currency Name</Attribute>
+        </Attributes>
+        <Icon>
+          CryptoWallet.svg
+        </Icon>
+    </Crypto_Wallet>
+</Financials>

Core/Entities/Groups.xml

@@ -0,0 +1,46 @@
+<Collectives>
+    <Company>
+        <Attributes>
+            <Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Registration Number</Attribute>
+        </Attributes>
+        <Icon>
+            Company.svg
+        </Icon>
+    </Company>
+    <Organization>
+        <Attributes>
+            <Attribute default="Organization Name" check="String" primary="True">Organization Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Registration Number</Attribute>
+        </Attributes>
+        <Icon>
+            Organization.svg
+        </Icon>
+    </Organization>
+    <Cult>
+        <Attributes>
+            <Attribute default="Default Cult" check="String" primary="True">Cult Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Ideology</Attribute>
+        </Attributes>
+        <Icon>
+            Cult.svg
+        </Icon>
+    </Cult>
+    <Terrorist_Group>
+        <Attributes>
+            <Attribute default="Terror Cell Name" check="String" primary="True">Group Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Ideology</Attribute>
+        </Attributes>
+        <Icon>
+            Terrorists.svg
+        </Icon>
+    </Terrorist_Group>
+    <Family>
+        <Attributes>
+            <Attribute default="Mr. and Mrs. Smith" check="String" primary="True">Family Surname</Attribute>
+        </Attributes>
+        <Icon>
+            Family.svg
+        </Icon>
+    </Family>
+</Collectives>

Core/Entities/Identifiers.xml

@@ -0,0 +1,35 @@
+<Identifiers>
+    <Identification_Number>
+        <Attributes>
+            <Attribute default="ID Number" check="String" primary="True">ID Number</Attribute>
+        </Attributes>
+        <Icon>
+            ID_Number.svg
+        </Icon>
+    </Identification_Number>
+    <Passport_Number>
+        <Attributes>
+            <Attribute default="Passport Number" check="String" primary="True">Passport Number</Attribute>
+        </Attributes>
+        <Icon>
+            Passport.svg
+        </Icon>
+    </Passport_Number>
+    <Vehicle_Registration>
+        <Attributes>
+            <Attribute default="Vehicle Registration Number" check="String" primary="True">Registration Number</Attribute>
+        </Attributes>
+        <Icon>
+            VehicleRegistration.svg
+        </Icon>
+    </Vehicle_Registration>
+    <Flight_Number>
+        <Attributes>
+            <Attribute default="Flight Number" check="String" primary="True">Flight Number</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Airline</Attribute>
+        </Attributes>
+        <Icon>
+            Flight_Number.svg
+        </Icon>
+    </Flight_Number>
+</Identifiers>

Core/Entities/Individuals.xml

@@ -0,0 +1,66 @@
+<Individuals>
+  <Person>
+    <Attributes>
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
+    </Attributes>
+    <Icon>
+      Person.svg
+    </Icon>
+  </Person>
+  <Politically_Exposed_Person>
+    <Attributes>
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
+    </Attributes>
+    <Icon>
+      PEP.svg
+    </Icon>
+  </Politically_Exposed_Person>
+  <Phrase>
+    <Attributes>
+      <Attribute default="Phrase" check="String" primary="True">Phrase</Attribute>
+    </Attributes>
+    <Icon>
+      Phrase.svg
+    </Icon>
+  </Phrase>
+  <Sentiment>
+    <Attributes>
+      <Attribute default="Positive" check="String" primary="True">Sentiment</Attribute>
+    </Attributes>
+    <Icon>
+      Sentiment.svg
+    </Icon>
+  </Sentiment>
+  <Email_Address>
+    <Attributes>
+      <Attribute default="example@example.com" check="Email" primary="True">Email Address</Attribute>
+    </Attributes>
+    <Icon>
+      Email.svg
+    </Icon>
+  </Email_Address>
+  <Phone_Number>
+    <Attributes>
+      <Attribute default="911" check="Phonenumber" primary="True">Phone Number</Attribute>
+    </Attributes>
+    <Icon>
+      Phone_Number.svg
+    </Icon>
+  </Phone_Number>
+  <Date>
+    <Attributes>
+      <Attribute default="1997/1/1" check="String" primary="True">Date</Attribute>
+    </Attributes>
+    <Icon>
+      Date.svg
+    </Icon>
+  </Date>
+</Individuals>

Core/Entities/Infrastructure.xml

@@ -0,0 +1,101 @@
+<Infrastructure>
+  <Operating_System>
+    <Attributes>
+      <Attribute default="Operating System" check="String" primary="True">OS Name</Attribute>
+    </Attributes>
+    <Icon>
+      OperatingSystem.svg
+    </Icon>
+  </Operating_System>
+  <Network>
+    <Attributes>
+      <Attribute default="127.0.0.0" check="String" primary="True">IP Address</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Range</Attribute>
+    </Attributes>
+    <Icon>
+      Network.svg
+    </Icon>
+  </Network>
+  <Website>
+    <Attributes>
+      <Attribute default="https://example.com" check="URL" primary="True">URL</Attribute>
+    </Attributes>
+    <Icon>
+      Website.svg
+    </Icon>
+  </Website>
+  <Onion_Website>
+    <Attributes>
+      <Attribute default="http://exampleexampleexampleexampleexampleexampleexampleexample.onion" check="Onion" primary="True">Onion URL</Attribute>
+    </Attributes>
+    <Icon>
+      OnionWebsite.svg
+    </Icon>
+  </Onion_Website>
+  <Domain>
+    <Attributes>
+      <Attribute default="example.com" check="Domain" primary="True">Domain Name</Attribute>
+    </Attributes>
+    <Icon>
+      Domain.svg
+    </Icon>
+  </Domain>
+  <IP_Address>
+    <Attributes>
+      <Attribute default="127.0.0.1" check="IPv4" primary="True">IP Address</Attribute>
+    </Attributes>
+    <Icon>
+      IP_Address.svg
+    </Icon>
+  </IP_Address>
+  <IPv6_Address>
+    <Attributes>
+      <Attribute default="::1" check="IPv6" primary="True">IPv6 Address</Attribute>
+    </Attributes>
+    <Icon>
+      IPv6_Address.svg
+    </Icon>
+  </IPv6_Address>
+  <Autonomous_System>
+    <Attributes>
+      <Attribute default="AS0" check="ASN" primary="True">AS Number</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">ASN Cidr</Attribute>
+    </Attributes>
+    <Icon>
+      AutonomousSystem.svg
+    </Icon>
+  </Autonomous_System>
+  <Port>
+    <Attributes>
+      <Attribute default="https://example.com:443" check="String" primary="True">Port</Attribute>
+    </Attributes>
+    <Icon>
+      Port.svg
+    </Icon>
+  </Port>
+  <MAC_Address>
+    <Attributes>
+      <Attribute default="AA:AA:AA:AA:AA:AA" check="MAC" primary="True">MAC Address</Attribute>
+    </Attributes>
+    <Icon>
+      MAC_Address.svg
+    </Icon>
+  </MAC_Address>
+  <Website_Infrastructure>
+    <Attributes>
+      <Attribute default="Infrastructure" check="String" primary="True">Infrastructure</Attribute>
+    </Attributes>
+    <Icon>
+      WebInfrastructure.svg
+    </Icon>
+  </Website_Infrastructure>
+  <Web_Certificate>
+    <Attributes>
+      <Attribute default="Subdomain" check="String" primary="True">Subdomain</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Authority</Attribute>
+    </Attributes>
+    <Icon>
+      Web_Certificate.svg
+    </Icon>
+  </Web_Certificate>
+</Infrastructure>

Core/Entities/Materials.xml

@@ -0,0 +1,56 @@
+<Materials>
+  <Document>
+    <Attributes>
+      <Attribute default="Document" check="String" primary="True">Document Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Document.svg
+    </Icon>
+  </Document>
+  <Spreadsheet>
+    <Attributes>
+      <Attribute default="Spreadsheet" check="String" primary="True">Spreadsheet Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Spreadsheet.svg
+    </Icon>
+  </Spreadsheet>
+  <Image>
+    <Attributes>
+      <Attribute default="Image" check="String" primary="True">Image Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Image.svg
+    </Icon>
+  </Image>
+  <Video>
+    <Attributes>
+      <Attribute default="Video" check="String" primary="True">Video Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Video.svg
+    </Icon>
+  </Video>
+  <Archive>
+    <Attributes>
+      <Attribute default="Archive" check="String" primary="True">Archive Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Archive.svg
+    </Icon>
+  </Archive>
+  <Disk>
+    <Attributes>
+      <Attribute default="Disk" check="String" primary="True">Disk Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      HardDisk.svg
+    </Icon>
+</Disk>
+</Materials>

Core/Entities/Meta.xml

@@ -0,0 +1,11 @@
+<Meta>
+  <EntityGroup>
+    <Attributes>
+      <Attribute default="Entity Group" check="String" primary="True">Group Name</Attribute>
+      <Attribute default="[]" check="None" primary="False">Child UIDs</Attribute>
+    </Attributes>
+    <Icon>
+      GroupNode.svg
+    </Icon>
+  </EntityGroup>
+</Meta>

Core/Entities/Places.xml

@@ -0,0 +1,50 @@
+<Places>
+  <GeoCoordinates>
+    <Attributes>
+      <Attribute default="Location Label" check="String" primary="True">Label</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Latitude</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Longitude</Attribute>
+    </Attributes>
+    <Icon>
+      Geolocation.svg
+    </Icon>
+  </GeoCoordinates>
+  <Address>
+    <Attributes>
+      <Attribute default="Street Address" check="String" primary="True">Street Address</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Locality</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Postal Code</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Country</Attribute>
+    </Attributes>
+    <Icon>
+      Address.svg
+    </Icon>
+  </Address>
+  <Country>
+    <Attributes>
+      <Attribute default="Country Name" check="String" primary="True">Country Name</Attribute>
+    </Attributes>
+    <Icon>
+      Country.svg
+    </Icon>
+  </Country>
+  <City>
+    <Attributes>
+      <Attribute default="City Name" check="String" primary="True">City Name</Attribute>
+    </Attributes>
+    <Icon>
+      City.svg
+    </Icon>
+  </City>
+  <Circular_Area>
+    <Attributes>
+      <Attribute default="Area Label" check="String" primary="True">Label</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Latitude</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Longitude</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Radius</Attribute>
+    </Attributes>
+    <Icon>
+      GeoArea.svg
+    </Icon>
+  </Circular_Area>
+</Places>

Core/Entities/Social Media.xml

@@ -0,0 +1,18 @@
+<Social>
+    <Social_Media_Group>
+        <Attributes>
+            <Attribute default="Guest Group" check="String" primary="True">Group Name</Attribute>
+        </Attributes>
+        <Icon>
+            Social_Media_Group.svg
+        </Icon>
+    </Social_Media_Group>
+    <Social_Media_Handle>
+        <Attributes>
+            <Attribute default="Guest123" check="String" primary="True">User Name</Attribute>
+        </Attributes>
+        <Icon>
+            Social_Media_Account.svg
+        </Icon>
+    </Social_Media_Handle>
+</Social>

Core/Entities/Software.xml

@@ -0,0 +1,28 @@
+<Software>
+    <Application>
+        <Attributes>
+            <Attribute default="Application Name" check="String" primary="True">Application Name</Attribute>
+        </Attributes>
+        <Icon>
+            Software.svg
+        </Icon>
+    </Application>
+    <Hash>
+        <Attributes>
+            <Attribute default="Hash Digest Value" check="String" primary="True">Hash Value</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Hash Algorithm</Attribute>
+        </Attributes>
+        <Icon>
+            Hash.svg
+        </Icon>
+    </Hash>
+    <CVE>
+        <Attributes>
+            <Attribute default="Unassigned CVE" check="String" primary="True">CVE</Attribute>
+            <Attribute default="0.0" check="Float" primary="False">CVSS Score</Attribute>
+        </Attributes>
+        <Icon>
+            CVE.svg
+        </Icon>
+    </CVE>
+</Software>

Core/Entities/Vehicles.xml

@@ -0,0 +1,24 @@
+<Vehicles>
+    <Vessel>
+        <Attributes>
+            <Attribute default="IMO 0" check="String" primary="True">Vessel Registration</Attribute>
+            <Attribute default="Vessel" check="String" primary="False">Vessel Name</Attribute>
+            <Attribute default="000000" check="String" primary="False">Vessel CallSign</Attribute>
+            <Attribute default="XX" check="String" primary="False">Vessel Flag</Attribute>
+        </Attributes>
+        <Icon>
+            Vessel.svg
+        </Icon>
+    </Vessel>
+    <Aircraft>
+        <Attributes>
+            <Attribute default="0000" check="String" primary="True">Aircraft Serial Number</Attribute>
+            <Attribute default="00-000" check="String" primary="False">Aircraft Registration</Attribute>
+            <Attribute default="Aircraft" check="String" primary="False">Aircraft Name</Attribute>
+            <Attribute default="Aircraft Model" check="String" primary="False">Aircraft Model</Attribute>
+        </Attributes>
+        <Icon>
+            Aircraft.svg
+        </Icon>
+    </Aircraft>
+</Vehicles>

Core/EntityDB.py

@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+
+from shutil import move
+from msgpack import load, dump
+from threading import Lock
+from pathlib import Path
+from typing import Union
+
+import networkx as nx
+
+
+class EntitiesDB:
+    """
+    This is a class that handles the addition and removal of entities and
+    links on a project-wide scale.
+    """
+
+    def __init__(self, mainWindow, messageHandler, resourceHandler) -> None:
+        self.messageHandler = messageHandler
+        self.resourceHandler = resourceHandler
+        self.mainWindow = mainWindow
+        self.dbLock = Lock()
+        self.database = None
+
+        self.loadDatabase()
+        self.resetTimeline()
+
+    def loadDatabase(self) -> None:
+        """
+        Load DiGraph from LinkScope Database file - msgpack dumped object.
+        """
+        with self.dbLock:
+            if self.database is not None:
+                self.save()
+            databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.lsdb")
+            self.messageHandler.debug(f'Opening Database at: {str(databaseFile)}')
+            try:
+                with open(databaseFile, "rb") as dbFile:
+                    self.database = self.mainWindow.RESOURCEHANDLER.reconstructGraphFullFromFile(load(dbFile))
+                self.messageHandler.info('Loaded Local Entities Database.')
+            except FileNotFoundError:
+                self.messageHandler.info('Creating new Local Entities Database.')
+                self.database = nx.DiGraph()
+            except Exception as exc:
+                self.messageHandler.error(f'Cannot parse Database: {str(exc)}\nCreating new Local Entities Database.',
+                                          popUp=True)
+                self.database = nx.DiGraph()
+
+    def resetTimeline(self) -> None:
+        """
+        Reset the timeline on dockBarThree to reflect the current state of the database.
+        """
+        with self.dbLock:
+            if self.database is not None:
+                self.mainWindow.resetTimeline(self.database)
+
+    def updateTimeline(self, node, added: bool, updateGraph: bool = True) -> None:
+        """
+        Update the timeline on dockBarThree to reflect the newest change of the database.
+        """
+        with self.dbLock:
+            self.mainWindow.updateTimeline(node, added, updateGraph)
+
+    def save(self) -> None:
+        """
+        Saves the graph to the specified file.
+        """
+
+        # Get the database file path again, in case it changed.
+        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.lsdb")
+        if databaseFile is None:
+            raise ValueError('Database File is None, cannot save database.')
+        with self.dbLock:
+            tmpSavePath = databaseFile.with_suffix(f'{databaseFile.suffix}.tmp')
+            with open(tmpSavePath, "wb") as dbFile:
+                dump(self.mainWindow.RESOURCEHANDLER.deconstructGraphForFileDump(self.database), dbFile)
+            move(tmpSavePath, databaseFile)
+            self.messageHandler.info('Database Saved.')
+
+    def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True) -> Union[dict, None]:
+        """
+        Adds the entity represented by the json dictionary to the database.
+        """
+        with self.dbLock:
+            returnValue = None
+
+            # Check if we're overwriting an existing entity
+            exists = None
+            if entJson.get('uid') is not None:
+                exists = self.getEntityNoLock(entJson.get('uid'))
+
+            entity = self.resourceHandler.getEntityJson(
+                entJson.get('Entity Type'),
+                entJson)
+
+            if entity is None:
+                return returnValue
+            # Use uid as key. Code is holdover from the time when primary field == uid.
+            self.database.add_node(entity['uid'], **entity)
+            returnValue = entity
+            if exists:
+                # Update canvases if the node already exists.
+                self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
+
+        if not fromServer:
+            self.mainWindow.sendLocalDatabaseUpdateToServer(entity, 1)
+        self.mainWindow.populateEntitiesWidget(returnValue, add=True)
+
+        if updateTimeline:
+            if exists is not None:
+                # Remove existing item before re-adding.
+                self.updateTimeline(exists, False, updateGraph=False)
+            self.updateTimeline(entity, True, updateGraph=True)
+
+        return returnValue
+
+    def addEntities(self, entitiesJsonList: Union[list, set, tuple], fromServer: bool = False) -> list:
+        with self.dbLock:
+            returnValue = []
+
+            for entJson in entitiesJsonList:
+                # Check if we're overwriting an existing entity
+                exists = None
+                if entJson.get('uid') is not None:
+                    exists = self.getEntityNoLock(entJson.get('uid'))
+
+                entity = self.resourceHandler.getEntityJson(
+                    entJson.get('Entity Type'),
+                    entJson)
+
+                if entity is None:
+                    continue
+                # Use uid as key. Code is holdover from the time when primary field == uid.
+                self.database.add_node(entity['uid'], **entity)
+                returnValue.append(entity)
+                if exists:
+                    # Update canvases if the node already exists.
+                    self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
+                if not fromServer:
+                    self.mainWindow.sendLocalDatabaseUpdateToServer(entity, 1)
+                self.mainWindow.populateEntitiesWidget(entity, add=True)
+
+        self.resetTimeline()
+
+        return returnValue
+
+    def addLink(self, linkJson: dict, fromServer: bool = False, overwrite: bool = False) -> Union[dict, None]:
+        """
+        Add a link between two entities in the database.
+
+        :param overwrite:
+        :param linkJson:
+        :param fromServer:
+        :return:
+        """
+        with self.dbLock:
+            exists = self.isLinkNoLock(linkJson['uid'])
+            link = self.resourceHandler.getLinkJson(linkJson)
+            if link is None:
+                # This can technically be caused by a race condition if the user
+                #   either tries really hard or gets really unlucky.
+                # Caused by deleting a node faster than the link can be created.
+                self.messageHandler.error("Attempted to add Link with "
+                                          "no uid to database.", popUp=True)
+                return None
+            else:
+                linkUID = link['uid']
+                if exists:
+                    newRes = link.get('Resolution')
+                    newNotes = link.get('Notes')
+                    if newRes and newRes != exists['Resolution']:
+                        if overwrite:
+                            link['Resolution'] = newRes
+                        else:
+                            link['Resolution'] = f"{exists['Resolution']} | {newRes}"
+                    if newNotes and newNotes != exists['Notes'] and newNotes != 'None':
+                        if overwrite:
+                            link['Notes'] = str(newNotes)
+                        else:
+                            link['Notes'] = f"{exists['Notes']}\n\n{str(newNotes)}"
+                    exists.update(link)
+                    link.update(exists)
+                    # Update canvases if the link already exists.
+                    # We can do this before updating the database here because the GUI will be updated only after this
+                    #   function returns. If we ever execute this function outside the main event loop, we will need
+                    #   to alter the execution flow.
+                    self.mainWindow.updateLinkLabelsOnCanvases(f"{linkUID[0]}{linkUID[1]}", link['Resolution'])
+                self.database.add_edge(linkUID[0], linkUID[1], **link)
+
+        if not fromServer:
+            if overwrite:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(link, 3)
+            else:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(link, 1)
+        return link
+
+    def getEntity(self, uid: str) -> Union[dict, None]:
+        """
+        Returns the attributes of the given entity uid as a dict.
+        """
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.nodes[uid]
+            except KeyError:
+                self.messageHandler.warning(f"Tried to get entity with nonexistent UID: {uid}")
+            finally:
+                return returnValue
+
+    def getAllEntities(self) -> Union[None, list]:
+        """
+        Returns a list containing the Json representation of every entity in the database.
+        """
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = [self.database.nodes[node] for node in self.database.nodes()]
+            except KeyError:
+                self.messageHandler.error("Tried to get entity with nonexistent UID.")
+            finally:
+                return returnValue
+
+    def getAllLinks(self) -> Union[None, list]:
+        """
+        Returns a list containing the Json representation of every link in the database.
+        :return:
+        """
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = [self.database.edges[edge] for edge in self.database.edges()]
+            except KeyError:
+                self.messageHandler.error("Tried to get link with nonexistent UID.")
+            finally:
+                return returnValue
+
+    def getEntityNoLock(self, uid: str) -> Union[None, dict]:
+        """
+        Returns the attributes of the given entity uid as a dict.
+
+        Does not lock, specifically meant for use by other functions in this
+        class.
+        """
+        returnValue = None
+        try:
+            returnValue = self.database.nodes[uid]
+        except KeyError:
+            pass
+        finally:
+            return returnValue
+
+    def getLink(self, uid) -> Union[None, dict]:
+        """
+        Returns the attributes of the given link uid as a dict.
+        """
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.edges[uid]
+            except KeyError:
+                self.messageHandler.error(
+                    "Tried to get link with nonexistent UID.")
+            finally:
+                return returnValue
+
+    def removeEntity(self, uid: str, fromServer=False, updateTimeLine=True) -> None:
+        """
+        Removes the entity with the given uid, if it exists.
+        """
+        with self.dbLock:
+            ent = None
+            if self.isNodeNoLock(uid):
+                ent = self.getEntityNoLock(uid)
+                self.mainWindow.populateEntitiesWidget(ent, add=False)
+                self.database.remove_node(uid)
+
+        if ent is not None:
+            self.mainWindow.handleGroupNodeUpdateAfterEntityDeletion(uid)  # Blocking - locks the db.
+            if not fromServer:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(ent, 2)
+            if updateTimeLine:
+                self.updateTimeline(ent, False)
+
+    def removeLink(self, uid, fromServer=False) -> None:
+        """
+        Removes the link with the given uid (in string or tuple form),
+        if it exists.
+        """
+        with self.dbLock:
+            if self.isLinkNoLock(uid):
+                self.database.remove_edge(uid[0], uid[1])
+        if not fromServer:
+            self.mainWindow.sendLocalDatabaseUpdateToServer({"uid": uid}, 2)
+
+    def doesEntityExist(self, primaryAttr: str) -> bool:
+        """
+        Checks if an entity with the specified primary attribute exists.
+        """
+        with self.dbLock:
+            result = False
+            for node in self.database.nodes():
+                details = self.database.nodes[node]
+                if details[list(details)[1]] == primaryAttr:
+                    result = True
+                    break
+        return result
+
+    def getEntityOfType(self, primaryAttr: str, entityType: str) -> Union[dict, None]:
+        """
+        Checks if an entity with the specified primary attribute exists, and if it does, return it.
+        """
+        result = None
+        primaryField = self.resourceHandler.getPrimaryFieldForEntityType(entityType)
+        if primaryField is None:
+            return result
+        with self.dbLock:
+            for node in self.database.nodes():
+                details = self.database.nodes[node]
+                if details['Entity Type'] == entityType and details[primaryField] == primaryAttr:
+                    result = dict(details)
+                    break
+        return result
+
+    def getLinkIfExists(self, uid) -> Union[None, dict]:
+        """
+        Returns the attributes of the given link uid as a dict.
+        Does not create an error if the link does not exist.
+        """
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.edges[uid]
+            except KeyError:
+                pass
+            finally:
+                return returnValue
+
+    def getIncomingLinks(self, uid: str):
+        """
+        Get all incoming edges for the given entity uid (primary attribute).
+        """
+        with self.dbLock:
+            returnValue = self.database.in_edges(uid) if self.isNodeNoLock(uid) else None
+        return returnValue
+
+    def getOutgoingLinks(self, uid: str):
+        """
+        Get all outgoing edges for the given entity uid (primary attribute).
+        """
+        with self.dbLock:
+            returnValue = self.database.out_edges(uid) if self.isNodeNoLock(uid) else None
+        return returnValue
+
+    def isNode(self, uid: Union[str, list, tuple]) -> bool:
+        """
+        Returns True if the uid (primary attribute) given exists as
+        an entity, and False otherwise.
+        """
+        with self.dbLock:
+            returnValue = isinstance(uid, str) and self.database.nodes.get(uid) is not None
+        return returnValue
+
+    def isNodeNoLock(self, uid: str) -> bool:
+        """
+        Returns True if the uid (primary attribute) given exists as
+        an entity, and False otherwise.
+
+        Used only in this class, as it does not lock.
+        """
+        return self.database.nodes.get(uid) is not None
+
+    def isLink(self, uid: Union[str, list, tuple]) -> bool:
+        """
+        Returns True if the uid given exists as a link, and False otherwise.
+        """
+        with self.dbLock:
+            returnValue = isinstance(uid, tuple) and self.database.edges.get(uid) is not None
+        return returnValue
+
+    def isLinkNoLock(self, uid: tuple) -> Union[bool, dict]:
+        """
+        Returns True if the uid given exists as a link, and False otherwise.
+        
+        Used only in this class, as it does not lock.
+        """
+        if self.database.edges.get(uid) is not None:
+            return self.database.edges[uid]
+        return False
+
+    def getEntityType(self, uid: str) -> Union[None, dict]:
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.getEntityNoLock(uid)['Entity Type']
+            except KeyError:
+                pass
+            finally:
+                return returnValue
+
+    def mergeDatabases(self, newDB_nodes: dict, newDB_edges: dict, fromServer=True) -> None:
+        """
+        Merges the existing database with the one provided.
+        Overwrites older attributes with newer ones based on date last edited.
+        """
+        with self.dbLock:
+            differenceGraph = nx.DiGraph()
+            differenceGraph.add_nodes_from([(n, nDict)
+                                            for n, nDict in newDB_nodes.items() if (n not in self.database.nodes()) or
+                                            (
+                                                    nDict.get('Date Last Edited', '') >
+                                                    self.database.nodes[n].get('Date Last Edited', '')
+                                            )
+                                            ])
+            differenceGraph.add_edges_from([(e[0], e[1], eDict)
+                                            for e, eDict in newDB_edges.items() if (e not in self.database.edges()) or
+                                            (
+                                                    eDict.get('Date Last Edited', '') >
+                                                    self.database.edges[e].get('Date Last Edited', '')
+                                            )
+                                            ])
+            if differenceGraph.number_of_nodes():
+                self.database = nx.compose(self.database, differenceGraph)
+                # Some nodes given by differenceGraph may be empty dicts, with an existing node's uid as the key.
+                for node in differenceGraph.nodes:
+                    self.mainWindow.populateEntitiesWidget(self.database.nodes[node], add=True)
+
+                if not fromServer and self.mainWindow.FCOM.isConnected():
+                    # Assume we are already synced with server, so just send the difference.
+                    self.mainWindow.FCOM.syncDatabase(self.mainWindow.SETTINGS.value("Project/Server/Project"),
+                                                      differenceGraph)

Core/FrontendCommunicationsHandler.py

@@ -0,0 +1,831 @@
+#!/usr/bin/env python3
+
+
+import contextlib
+from ast import literal_eval
+from typing import Union
+from pathlib import Path
+from msgpack import loads, dumps
+import socket
+import threading
+import time
+import re
+import networkx as nx
+
+from queue import Queue, Empty
+from base64 import b64encode, b64decode
+from PySide6 import QtCore
+from uuid import uuid4
+from hashlib import sha3_512
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.serialization import Encoding
+from cryptography.hazmat.primitives.serialization import PublicFormat
+from cryptography.hazmat.primitives.serialization import load_der_public_key
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.exceptions import InvalidTag
+
+# Amount of data to place in each message
+MESSAGE_DATA_SIZE = 8192
+
+# Needs to be a bit bigger than MESSAGE_DATA_SIZE
+RECV_SIZE = MESSAGE_DATA_SIZE + 1024
+
+# All functions use this to run. Checks if the program is closing.
+closeSoftwareLock = threading.Lock()
+closeSoftware = False
+
+
+# Has to be QObject for Signals to work.
+class CommunicationsHandler(QtCore.QObject):
+    """
+    This class handles communication with a LinkScope server.
+
+    Convention for variables changes here - all variables that have to do with network communications
+      are in snake_case. This makes them easy to differentiate from others, if it is important to track
+      what came from a network and what did not. This is also the case server-side.
+    """
+
+    connected_to_server_listener = QtCore.Signal(str)
+    receive_question_answer = QtCore.Signal(dict)
+    receive_chat_message = QtCore.Signal(str)
+    receive_collectors_signal = QtCore.Signal(dict, dict)
+    receive_start_collector_signal = QtCore.Signal(str, str, str, list, dict)
+    receive_collector_result_signal = QtCore.Signal(str, str, str, list)
+    receive_resolutions_signal = QtCore.Signal(dict)
+    receive_completed_resolution_result_signal = QtCore.Signal(str, list, str)
+    receive_completed_resolution_string_result_signal = QtCore.Signal(str, str, str)
+    receive_document_summary_signal = QtCore.Signal(str, str)
+    remove_server_resolution_from_running_signal = QtCore.Signal(str)
+    receive_projects_list_signal = QtCore.Signal(list)
+    delete_server_project_signal = QtCore.Signal(str)
+    receive_project_canvases_list_signal = QtCore.Signal(list)
+    open_project_signal = QtCore.Signal(str)
+    close_project_signal = QtCore.Signal()
+    open_project_canvas_signal = QtCore.Signal(str)
+    close_project_canvas_signal = QtCore.Signal(str)
+    receive_project_database_update = QtCore.Signal(dict, int)
+    receive_project_canvas_update_node = QtCore.Signal(str, str)
+    receive_project_canvas_update_link = QtCore.Signal(str, tuple)
+    receive_sync_database = QtCore.Signal(dict, dict)
+    status_message_signal = QtCore.Signal(str, bool)
+    receive_project_file_list = QtCore.Signal(list)
+    file_upload_finished_signal = QtCore.Signal(str)
+    file_upload_abort_signal = QtCore.Signal(str)
+    receive_sync_canvas_signal = QtCore.Signal(str, dict, dict)
+
+    def __init__(self, mainWindow):
+
+        super().__init__()
+
+        self.inbox = Queue()
+
+        self.mainWindow = mainWindow
+        self.downloadingFiles = {}
+        self.uploadingFiles = {}
+
+        self.connected_to_server_listener.connect(self.mainWindow.connectedToServerListener)
+        self.receive_question_answer.connect(self.mainWindow.questionAnswerListener)
+        self.receive_chat_message.connect(self.mainWindow.receiveChatMessage)
+        self.receive_collectors_signal.connect(self.mainWindow.addCollectorsFromServerListener)
+        self.receive_start_collector_signal.connect(self.mainWindow.startNewCollectorListener)
+        self.receive_collector_result_signal.connect(self.mainWindow.receiveCollectorResultListener)
+        self.receive_resolutions_signal.connect(self.mainWindow.addResolutionsFromServerListener)
+        self.receive_completed_resolution_result_signal.connect(self.mainWindow.resolutionSignalListener)
+        self.receive_completed_resolution_string_result_signal.connect(self.mainWindow.resolutionSignalListener)
+        self.receive_document_summary_signal.connect(self.mainWindow.receiveSummaryOfDocument)
+        self.remove_server_resolution_from_running_signal.connect(self.mainWindow.cleanServerResolutionListener)
+        self.delete_server_project_signal.connect(self.mainWindow.receiveProjectDeleteListener)
+        self.receive_projects_list_signal.connect(self.mainWindow.receiveProjectsListListener)
+        self.receive_project_canvases_list_signal.connect(self.mainWindow.receiveProjectCanvasesListListener)
+        self.open_project_signal.connect(self.mainWindow.openServerProjectListener)
+        self.open_project_canvas_signal.connect(self.mainWindow.openServerCanvasListener)
+        self.close_project_signal.connect(self.mainWindow.closeServerProjectListener)
+        self.close_project_canvas_signal.connect(self.mainWindow.closeServerCanvasListener)
+        self.receive_project_database_update.connect(self.mainWindow.receiveServerDatabaseUpdate)
+        self.receive_project_canvas_update_node.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_project_canvas_update_link.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_sync_database.connect(self.mainWindow.receiveSyncDatabaseListener)
+        self.receive_project_file_list.connect(self.mainWindow.receiveFileListListener)
+        self.file_upload_finished_signal.connect(self.mainWindow.fileUploadFinishedListener)
+        self.file_upload_abort_signal.connect(self.mainWindow.receiveAbortUploadOfFiles)
+        self.receive_sync_canvas_signal.connect(self.mainWindow.receiveSyncCanvasListener)
+        self.status_message_signal.connect(self.mainWindow.statusMessageListener)
+
+        self.cipher = None
+        self.threadInc = None
+        self.threadInb = None
+        self.sock = None
+
+    def beginCommunications(self, password: str, server: str, port: int = 3777) -> bool:
+        global closeSoftwareLock
+        global closeSoftware
+        with closeSoftwareLock:  # Not strictly necessary, but might as well just in case.
+            closeSoftware = False
+        try:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+            self.sock.connect((server, port))
+
+            private_key = ec.generate_private_key(ec.SECP521R1())
+            self.sock.send(private_key.public_key().public_bytes(
+                Encoding.DER,
+                format=PublicFormat.SubjectPublicKeyInfo))
+            peer_public = load_der_public_key(self.sock.recv(RECV_SIZE))
+            shared_key = private_key.exchange(ec.ECDH(), peer_public)
+            derived_key = HKDF(
+                algorithm=hashes.SHA3_256(),
+                length=32,
+                salt=None,
+                info=b'Handshake'
+            ).derive(shared_key)
+            iv = sha3_512(derived_key).digest()[:16]
+            self.cipher = Cipher(algorithms.AES(derived_key), modes.CBC(iv))
+            encryptor = self.cipher.encryptor()
+            decrypter = self.cipher.decryptor()
+
+            passPad = 16 - (len(password) % 16)
+            passMessage = encryptor.update(password.encode() + b"*" * passPad) + encryptor.finalize()
+            self.sock.send(passMessage)
+            messageReceived = decrypter.update(self.sock.recv(RECV_SIZE)) + decrypter.finalize()
+            if messageReceived == b"Passphrase is OK":
+                self.threadInc = threading.Thread(target=self.scanIncoming)
+                self.threadInc.setDaemon(True)
+                self.threadInc.start()
+
+                self.threadInb = threading.Thread(target=self.scanInbox)
+                self.threadInb.setDaemon(True)
+                self.threadInb.start()
+
+                return True
+            elif messageReceived == b"Wrong Passphrase":
+                raise ValueError('Incorrect Passphrase')
+
+            self.mainWindow.MESSAGEHANDLER.warning("Server did not reply in the expected manner to "
+                                                   "password authentication.", popUp=True)
+        except ConnectionRefusedError:
+            self.mainWindow.MESSAGEHANDLER.error("Did not connect: Server not running.", popUp=True, exc_info=False)
+        except Exception as exception:
+            self.mainWindow.MESSAGEHANDLER.error(f"Did not connect: {str(exception)}")
+
+        try:
+            if self.sock is not None:
+                self.sock.shutdown(socket.SHUT_RDWR)
+        except OSError:
+            # This would typically occur if the socket is already shut down.
+            self.mainWindow.MESSAGEHANDLER.debug('Tried to shutdown socket that was already shut down.')
+        try:
+            self.sock.close()
+        finally:
+            self.sock = None
+
+        return False
+
+    def isConnected(self) -> bool:
+        """
+        Check if socket is in a working state.
+        """
+        return self.sock is not None and self.sock.fileno() != -1
+
+    def close(self) -> None:
+        global closeSoftwareLock
+        global closeSoftware
+        with closeSoftwareLock:
+            closeSoftware = True
+        self.closeSocket()
+        for fileName in self.downloadingFiles:
+            # Close open file handlers
+            self.downloadingFiles[fileName].close()
+            # saveDir = Path(fileName)
+            saveDir = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / fileName
+            if saveDir.exists():
+                # Delete partly downloaded files.
+                saveDir.unlink(missing_ok=True)
+        self.downloadingFiles = {}
+        self.uploadingFiles = {}
+
+    def encryptTransmission(self, bytesObject) -> bytes:
+        encryptor = self.cipher.encryptor()
+        padNeeded = 16 - (len(bytesObject) % 16)
+        message = encryptor.update(b'a' * padNeeded + bytesObject) + encryptor.finalize()
+        return b64encode(message)
+
+    def decryptTransmission(self, bytesObject) -> Union[bytes, None]:
+        decrypter = self.cipher.decryptor()
+        try:
+            return decrypter.update(bytesObject) + decrypter.finalize()
+        except InvalidTag:
+            # If the ciphertext cannot be decrypted to a valid message, return None.
+            return None
+
+    def transmitMessage(self, messageJson: dict, showErrorOnBrokenPipe: bool = True) -> None:
+        self.mainWindow.MESSAGEHANDLER.debug(f'Sending Message: {messageJson}')
+        argEncoded = str(messageJson)
+        largeMessageUUID = str(uuid4())
+        try:
+            for data in range(0, len(argEncoded), MESSAGE_DATA_SIZE):
+                partArg = argEncoded[data:data + MESSAGE_DATA_SIZE]
+                done = data + MESSAGE_DATA_SIZE >= len(argEncoded)
+                messageJson = {"uuid": largeMessageUUID,
+                               "message": partArg,
+                               "done": done}
+                self.sock.send(self.encryptTransmission(dumps(messageJson)) + b'\x03\x03\x03\x03\x03')
+        except BrokenPipeError:
+            if showErrorOnBrokenPipe:
+                self.mainWindow.MESSAGEHANDLER.error("Disconnected from server!", popUp=True, exc_info=False)
+                self.mainWindow.disconnectFromServer()
+
+    def closeSocket(self) -> None:
+        """
+        Called by the close function in this class when communications with the server are to be ended.
+        :return:
+        """
+        try:
+            self.transmitMessage({"Operation": "Close Socket", "Arguments": {}}, showErrorOnBrokenPipe=False)
+        except (OSError, AttributeError):
+            # OSError thrown due to bad file descriptor, i.e. server is closed.
+            # AttributeError thrown if no connection was established while the software was running
+            pass
+        finally:
+            with contextlib.suppress(OSError):
+                # OSError thrown if the socket is already closed.
+                if self.sock is not None:
+                    self.sock.shutdown(socket.SHUT_RDWR)
+            try:
+                self.sock.close()
+            finally:
+                self.sock = None
+
+    def scanIncoming(self) -> None:
+        """
+        This function listens for incoming data, and puts it in the queue if
+        it exists.
+        """
+        preInbox = {}
+        oldData = b''
+        while True:
+            try:
+                receivedInfo = self.sock.recv(RECV_SIZE)
+                if receivedInfo == b'':
+                    # Socket closed.
+                    break
+                receivedInfo = oldData + receivedInfo
+                messages = receivedInfo.split(b'\x03\x03\x03\x03\x03')
+                # Last message is either blank (i.e. '') or incomplete data, so we ignore it.
+                oldData = messages[-1]
+                messages = messages[:-1]
+                for message in messages:
+                    if len(message) == 0:
+                        continue
+                    message = b64decode(message)
+                    decryptedInfo = self.decryptTransmission(message)
+                    if decryptedInfo is None:
+                        self.mainWindow.MESSAGEHANDLER.warning("Invalid message received from server.")
+                        continue
+                    receivedMessage = loads(re.sub(b'^a*', b'', decryptedInfo, count=1))
+                    messageID = receivedMessage.get("uuid")
+                    if messageID in preInbox:
+                        preInbox[messageID]["message"] += receivedMessage.get("message")
+                    else:
+                        preInbox[messageID] = receivedMessage
+                    if receivedMessage.get("done"):
+                        preInbox[messageID]["message"] = literal_eval(preInbox[messageID]["message"])
+                        self.inbox.put(preInbox.pop(messageID).get("message"))
+
+            except socket.error as socketError:
+                self.mainWindow.MESSAGEHANDLER.error(f'Socket Error: {str(socketError)}')
+                # If something happens, wait 2 seconds then try again.
+                with closeSoftwareLock:
+                    if closeSoftware:
+                        break
+                time.sleep(2)
+            except ValueError:
+                # E.g.: Unpack failed: incomplete input
+                # In this case, we are being sent fragmented messages.
+                # They will be reconstructed eventually, once all the pieces get here.
+                continue
+
+    def askServerForCollectors(self, continuing_collectors_dict: dict) -> None:
+        message = {"Operation": "Get Server Collectors",
+                   "Arguments": {
+                       'continuing_collectors_dict': continuing_collectors_dict
+                   }}
+        self.transmitMessage(message)
+
+    def receiveStartCollector(self, collector_category: str, collector_name: str, collector_uid: str,
+                              collector_entities: list, collector_parameters: dict):
+        self.receive_start_collector_signal.emit(collector_category, collector_name, collector_uid,
+                                                 collector_entities, collector_parameters)
+
+    def receiveCollectors(self, server_collectors: dict, continuing_collectors_info: dict) -> None:
+        self.receive_collectors_signal.emit(server_collectors, continuing_collectors_info)
+
+    def startServerCollector(self, collector_name: str, collector_entities: list, collector_parameters: dict,
+                             continueTimestamp: int = 0) -> None:
+        collector_entities_to_send = []
+        for entity in collector_entities:
+            with contextlib.suppress(KeyError):
+                dereferenced_entity = dict(entity)
+                collector_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any collector as of now: 2022/4/3.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+        message = {'Operation': 'Start Server Collector',
+                   'Arguments': {
+                       'collector_name': collector_name,
+                       'collector_entities': collector_entities_to_send,
+                       'collector_parameters': collector_parameters,
+                       'continue_time': continueTimestamp
+                   }}
+        self.transmitMessage(message)
+
+    def stopServerCollector(self, collector_uid: str) -> None:
+        message = {"Operation": "Stop Server Collector",
+                   "Arguments": {
+                       'collector_uid': collector_uid
+                   }}
+        self.transmitMessage(message)
+
+    def receiveCollectorResult(self, collector_name: str, collector_uid: str, timestamp: str, results: list) -> None:
+        self.receive_collector_result_signal.emit(collector_name, collector_uid, timestamp, results)
+
+    def askServerForResolutions(self) -> None:
+        message = {"Operation": "Get Server Resolutions",
+                   "Arguments": {}}
+        self.transmitMessage(message)
+
+    def receiveResolutions(self, server_resolutions) -> None:
+        self.receive_resolutions_signal.emit(server_resolutions)
+
+    def runRemoteResolution(self, resolution_name: str, resolution_entities: list, resolution_parameters: dict,
+                            resolution_uid: str) -> None:
+        resolution_entities_to_send = []
+        for entity in resolution_entities:
+            with contextlib.suppress(KeyError):
+                dereferenced_entity = dict(entity)
+                resolution_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any resolution as of now: 2022/1/2.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+        message = {'Operation': 'Run Resolution',
+                   'Arguments': {
+                       'resolution_name': resolution_name,
+                       'resolution_entities': resolution_entities_to_send,
+                       'resolution_parameters': resolution_parameters,
+                       'resolution_uid': resolution_uid
+                   }}
+        self.transmitMessage(message)
+
+    def receiveResolutionResult(self, resolution_name: str, resolution_result: Union[list, str],
+                                resolution_uid: str) -> None:
+        if isinstance(resolution_result, str):
+            self.receive_completed_resolution_string_result_signal.emit(resolution_name, resolution_result,
+                                                                        resolution_uid)
+        else:
+            self.receive_completed_resolution_result_signal.emit(resolution_name, resolution_result,
+                                                                 resolution_uid)
+        self.remove_server_resolution_from_running_signal.emit(resolution_uid)
+
+    def abortResolution(self, resolution_name: str, resolution_uid: str) -> None:
+        message = {'Operation': 'Abort Resolution',
+                   'Arguments': {
+                       'resolution_name': resolution_name,
+                       'resolution_uid': resolution_uid
+                   }}
+        self.transmitMessage(message)
+
+    def askProjectsList(self) -> None:
+        message = {'Operation': 'Get Projects List',
+                   'Arguments': {}}
+        self.transmitMessage(message)
+
+    def receiveProjectsList(self, projects: list) -> None:
+        self.receive_projects_list_signal.emit(projects)
+
+    def createProject(self, projectName: str, projectPassword: str) -> None:
+        if self.isConnected():
+            message = {'Operation': 'Create Project',
+                       'Arguments': {'project_name': projectName, 'password': projectPassword}}
+            self.transmitMessage(message)
+
+    def openProject(self, project_name: str, projectPassword: str) -> None:
+        if self.isConnected():
+            message = {'Operation': 'Open Project',
+                       'Arguments': {'project_name': project_name, 'password': projectPassword}}
+            self.transmitMessage(message)
+
+    def closeProject(self, project_name: str) -> None:
+        if self.isConnected():
+            message = {'Operation': 'Close Project',
+                       'Arguments': {'project_name': project_name}}
+            self.transmitMessage(message)
+
+    def askProjectCanvasesList(self, project_name: str) -> None:
+        if self.isConnected():
+            message = {'Operation': 'List Synced Canvases',
+                       'Arguments': {'project_name': project_name}}
+            self.transmitMessage(message)
+
+    def receiveProjectCanvasesList(self, canvases: list) -> None:
+        self.receive_project_canvases_list_signal.emit(canvases)
+
+    def askQuestion(self, project_name: str, question: str, reader_value: int, retriever_value: int,
+                    answer_count: int) -> None:
+        if self.isConnected():
+            message = {"Operation": "Ask Question",
+                       "Arguments": {
+                           'project_name': project_name,
+                           'question': question[:256],
+                           'reader_value': reader_value,
+                           'retriever_value': retriever_value,
+                           'answer_count': answer_count}}
+            self.transmitMessage(message)
+
+    def receiveQuestionAnswer(self, answer: list) -> None:
+        self.receive_question_answer.emit(answer)
+
+    def receiveTextMessage(self, chat_message: str) -> None:
+        self.receive_chat_message.emit(chat_message)
+
+    def sendTextMessage(self, project_name: str, chat_message: str) -> None:
+        if self.isConnected():
+            message = {"Operation": "Chat",
+                       "Arguments": {
+                           "project_name": project_name,
+                           "chat_message": chat_message[:1024]}}  # Cap messages to prevent spam.
+            self.transmitMessage(message)
+
+    def syncDatabase(self, project_name: str, client_project_graph: nx.DiGraph) -> None:
+        message = {'Operation': 'Sync Database',
+                   'Arguments': {
+                       'project_name': project_name,
+                       'client_project_graph': str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(
+                           client_project_graph))
+                   }}
+        self.transmitMessage(message)
+
+    def receiveSyncDatabase(self, database: str) -> None:
+        database_nodes, database_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraphFromString(database)
+        self.receive_sync_database.emit(database_nodes, database_edges)
+
+    def askServerForFileList(self, project_name: str) -> None:
+        message = {"Operation": "Get File List",
+                   "Arguments": {'project_name': project_name}}
+        self.transmitMessage(message)
+
+    def syncCanvasSend(self, project_name: str, canvas_name: str, canvas_graph: nx.DiGraph) -> None:
+        if self.isConnected():
+            message = {'Operation': "Sync Canvas",
+                       'Arguments': {
+                           'project_name': project_name,
+                           'canvas_name': canvas_name,
+                           "canvas_graph": str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(canvas_graph))}}
+            self.transmitMessage(message)
+
+    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: str) -> None:
+        graph_nodes, graph_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraphFromString(canvas_graph)
+        self.receive_sync_canvas_signal.emit(canvas_name, graph_nodes, graph_edges)
+
+    def closeCanvas(self, project_name: str, canvas_name: str) -> None:
+        if self.isConnected():
+            message = {'Operation': 'Close Canvas',
+                       'Arguments': {'project_name': project_name,
+                                     'canvas_name': canvas_name}}
+            self.transmitMessage(message)
+
+    # Items are sent both from the database and from the canvas.
+    # Items are added to the database from the canvas, but items can be
+    #   removed from the canvas without removing them from the database.
+    # Furthermore, one might want to add items to the the database, but not
+    #   to a particular canvas.
+    # Being verbose is better than prematurely optimizing for a few kbps of
+    #   network traffic.
+    def receiveDatabaseUpdateEvent(self, entity_json: dict, add: int) -> None:
+        with contextlib.suppress(KeyError):
+            entity_json['Icon'] = QtCore.QByteArray(b64decode(entity_json['Icon']))
+        self.receive_project_database_update.emit(entity_json, add)
+
+    def sendDatabaseUpdateEvent(self, project_name: str, entity_json: dict, add: int) -> None:
+        with contextlib.suppress(KeyError):
+            entity_json['Icon'] = entity_json['Icon'].toBase64().data()
+        message = {"Operation": "Update Project Entities",
+                   "Arguments": {
+                       'project_name': project_name,
+                       "entity_json": entity_json,
+                       "add": add}}
+        self.transmitMessage(message)
+
+    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]) -> None:
+        if isinstance(entity_or_link_uid, str):
+            self.receive_project_canvas_update_node.emit(canvas_name, entity_or_link_uid)
+        else:
+            self.receive_project_canvas_update_link.emit(canvas_name, entity_or_link_uid)
+
+    def sendCanvasUpdateEvent(self, project_name: str, canvas_name: str,
+                              entity_or_link_uid: Union[str, tuple]) -> None:
+        message = {"Operation": "Update Canvas Entities",
+                   "Arguments": {
+                       'project_name': project_name,
+                       "canvas_name": canvas_name,
+                       "entity_or_link_uid": entity_or_link_uid}}
+        self.transmitMessage(message)
+
+    def receiveFileList(self, file_list: list) -> None:
+        self.receive_project_file_list.emit(file_list)
+
+    def sendFile(self, project_name: str, file_name: str, filePath: Path) -> None:
+        """
+        Starts a thread, calling sendFileHelper to send the file specified.
+
+        :param project_name:
+        :param file_name:
+        :param filePath:
+        :return:
+        """
+        sendHelperThread = threading.Thread(target=self.sendFileHelper, args=(project_name, file_name, filePath))
+        sendHelperThread.setDaemon(True)
+        self.uploadingFiles[file_name] = sendHelperThread
+        sendHelperThread.start()
+
+    def sendFileHelper(self, project_name: str, file_name: str, filePath: Path) -> None:
+        """
+        Sends file in chunks to avoid loading the entire thing in memory.
+
+        :param project_name:
+        :param file_name:
+        :param filePath:
+        :return:
+        """
+        if not filePath.exists() or not filePath.is_file():
+            return
+        with open(filePath, 'rb') as fileHandler:
+            currThread = threading.currentThread()
+            while getattr(currThread, "continue_running", True):
+                filePart = fileHandler.read(512)
+                if not filePart:
+                    messageJson = {"Operation": "File Upload Done",
+                                   "Arguments": {
+                                       'project_name': project_name,
+                                       'file_name': file_name
+                                   }}
+                    self.transmitMessage(messageJson)
+                    break
+                messageJson = {"Operation": "File Upload",
+                               "Arguments": {
+                                   'project_name': project_name,
+                                   'file_name': file_name,
+                                   'file_contents': filePart
+                               }}
+                self.transmitMessage(messageJson)
+
+    def sendFileAbort(self, project_name: str, file_name: str) -> None:
+        with contextlib.suppress(KeyError):
+            uploadToAbort = self.uploadingFiles.pop(file_name)
+            uploadToAbort.continue_running = False
+            messageJson = {"Operation": "File Upload Abort",
+                           "Arguments": {
+                               'project_name': project_name,
+                               'file_name': file_name
+                           }}
+            self.transmitMessage(messageJson)
+
+    def scanInbox(self) -> None:
+        """
+        This function checks if there is anything in the inbox, and if
+        there is, calls the appropriate functions.
+        """
+        prevMesg = None
+        while True:
+            try:
+                message = self.inbox.get(timeout=0.2)
+            except Empty:
+                with closeSoftwareLock:
+                    if closeSoftware:
+                        return
+                    time.sleep(0.2)
+                    continue
+            if prevMesg == message:
+                # Same message, do not waste time handling.
+                continue
+            self.mainWindow.MESSAGEHANDLER.debug(f'Message to handle: {str(message)}')
+            operation = message['Operation']
+            arguments = message['Arguments']
+            if operation == 'Get Server Resolutions':
+                self.receiveResolutions(**arguments)
+            elif operation == 'Get Projects List':
+                self.receiveProjectsList(**arguments)
+            elif operation == 'Status Message':
+                self.handleStatusMessage(**arguments)
+            elif operation == 'List Synced Canvases':
+                self.receiveProjectCanvasesList(**arguments)
+            elif operation == 'Resolution Result':
+                self.receiveResolutionResult(**arguments)
+            elif operation == 'Collector Results Signal':
+                self.receiveCollectorResult(**arguments)
+            elif operation == "Chat":
+                self.receiveTextMessage(**arguments)
+            elif operation == "Sync Database":
+                self.receiveSyncDatabase(**arguments)
+            elif operation == "Sync Canvas":
+                self.receiveSyncCanvas(**arguments)
+            elif operation == "Answer Question":
+                self.receiveQuestionAnswer(**arguments)
+            elif operation == "Update Project Entities":
+                self.receiveDatabaseUpdateEvent(**arguments)
+            elif operation == "Update Canvas Entities":
+                self.receiveCanvasUpdateEvent(**arguments)
+            elif operation == "File List":
+                self.receiveFileList(**arguments)
+            elif operation == "File Download":
+                self.receiveFileListener(**arguments)
+            elif operation == "Get File Summary":
+                self.receiveFileSummaryListener(**arguments)
+            elif operation == "File Upload Abort":
+                self.receiveFileUploadAbort(**arguments)
+            elif operation == "Get Server Collectors":
+                self.receiveCollectors(**arguments)
+            elif operation == "Start Collector":
+                self.receiveStartCollector(**arguments)
+            else:
+                self.mainWindow.MESSAGEHANDLER.warning(f'Unhandled message: {str(message)} On Operation: '
+                                                       f'{str(operation)}')
+            prevMesg = message
+
+    def handleStatusMessage(self, operation: str, message: str, status_code: int) -> None:
+        """
+        Operations that are completely server-side, or do not conform to the query - response model,
+        send status messages to inform the client of what is going on.
+
+        Status codes follow the same pattern as the ones in HTTP:
+        200 - Operation Successful.
+        404 - A requested resource was not found
+        403 - Access to the requested resource is forbidden
+        500 - Server could not adequately handle the request.
+        :param operation:
+        :param message:
+        :param status_code:
+        :return:
+        """
+        if status_code == 200:
+            if operation == 'Abort Resolution':
+                # Remove resolution from resolutions list.
+                resolution_uid = message.split(': ', 1)[1]
+                self.remove_server_resolution_from_running_signal.emit(resolution_uid)
+            elif operation == 'Close Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.close_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Close Project':
+                self.close_project_signal.emit()
+            elif operation == 'Connect To Server':
+                server_name = message.split(': ', 1)[1]
+                self.connected_to_server_listener.emit(server_name)
+            elif operation in {'Create Project', 'Create Canvas', 'Stop Collector'}:
+                # No need to do anything for these.
+                pass
+            elif operation == 'Delete Project':
+                # Remove project from server projects list.
+                project_name = message.split(': ', 1)[1]
+                self.delete_server_project_signal.emit(project_name)
+            elif operation == 'File Download Done':
+                file_name = message.split(': ', 1)[1]
+                self.receiveFileDoneListener(file_name)
+            elif operation == 'File Upload Abort':
+                file_name = message.split(': ', 1)[1]
+                # Remove file from uploading files list.
+                self.file_upload_abort_signal.emit(file_name)
+            elif operation == 'File Upload':
+                file_name = message.split(': ', 1)[1]
+                self.file_upload_finished_signal.emit(file_name)
+            elif operation == 'Open Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.open_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Open Project':
+                projectName = message.split(': ', 1)[1]
+                self.open_project_signal.emit(projectName)
+            elif operation == 'Opening Project':
+                self.status_message_signal.emit(message, True)
+            else:
+                self.mainWindow.MESSAGEHANDLER.warning(f'Unhandled status message: {message} Code: {status_code} '
+                                                       f'On Operation: {operation}')
+
+        elif status_code == 404 and message == 'No project with the specified name exists!':
+            self.close_project_signal.emit()
+        else:
+            self.status_message_signal.emit(f'Operation {operation} failed with status code {status_code}: {message}',
+                                            True)
+
+    def receiveFile(self, project_name: str, file_name: str, saveDir: Path) -> None:
+        # Do not download files already being downloaded.
+        if self.downloadingFiles.get(file_name) is None:
+            fileHandler = open(saveDir, "wb")
+            self.downloadingFiles[file_name] = fileHandler
+
+            message = {'Operation': 'File Download',
+                       'Arguments': {
+                           'project_name': project_name,
+                           'file_name': file_name
+                       }}
+            self.transmitMessage(message)
+
+    def receiveFileListener(self, file_name: str, file_contents: bytes) -> None:
+        try:
+            fileHandler = self.downloadingFiles.get(file_name)
+            fileHandler.write(file_contents)
+        except Exception:
+            # In case something goes wrong in the middle of writing.
+            self.mainWindow.MESSAGEHANDLER.warning(f'Received data for file: {file_name} but no valid file handler '
+                                                   f'exists for this file.')
+
+    def receiveFileDoneListener(self, file_name: str) -> None:
+        fileHandler = self.downloadingFiles.pop(file_name)
+        if fileHandler is None:
+            self.mainWindow.MESSAGEHANDLER.warning(f'Received file: {file_name} but no file handler exists for this '
+                                                   f'file.')
+
+            return
+
+        fileHandler.close()
+        self.status_message_signal.emit(f'Finished downloading file from server: {file_name}', True)
+
+    def receiveFileAbort(self, project_name: str, file_name: str) -> None:
+        messageJson = {"Operation": "File Download Abort",
+                       "Arguments": {
+                           'project_name': project_name,
+                           'file_name': file_name
+                       }}
+        self.transmitMessage(messageJson)
+        fileHandler = self.downloadingFiles.pop(file_name)
+        if fileHandler is None:
+            return
+        fileHandler.close()
+        abortedPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
+        abortedPath.unlink(missing_ok=True)
+
+    def deleteProject(self, project_name: str) -> None:
+        message = {"Operation": "Delete Project",
+                   "Arguments": {
+                       'project_name': project_name}}
+        self.transmitMessage(message)
+
+    def askServerForFileSummary(self, project_name: str, document_name: str) -> None:
+        message = {"Operation": "Get File Summary",
+                   "Arguments": {
+                       'project_name': project_name,
+                       'document_name': document_name}}
+        self.transmitMessage(message)
+
+    def receiveFileSummaryListener(self, document_name: str, summary: str) -> None:
+        self.receive_document_summary_signal.emit(document_name, summary)
+
+    def receiveFileUploadAbort(self, file_name: str) -> None:
+        """
+        If we are told by the server to stop uploading a file,
+        we should do so (i.e. because no space left on server),
+        to avoid wasting bandwidth.
+
+        :param file_name:
+        :return:
+        """
+        with contextlib.suppress(KeyError):
+            uploadToAbort = self.uploadingFiles.pop(file_name)
+            uploadToAbort.continue_running = False
+            self.file_upload_abort_signal.emit(file_name)
+
+    def sendFileAbortAll(self, project_name: str) -> None:
+        """
+        Abort the sending of all files currently being transmitted.
+
+        :param project_name:
+        :return:
+        """
+        for file_name in dict(self.uploadingFiles):
+            with contextlib.suppress(KeyError):
+                uploadToAbort = self.uploadingFiles.pop(file_name)
+                uploadToAbort.continue_running = False
+                messageJson = {"Operation": "File Upload Abort",
+                               "Arguments": {
+                                   'project_name': project_name,
+                                   'file_name': file_name
+                               }}
+                self.transmitMessage(messageJson)
+
+    def receiveFileAbortAll(self, project_name: str) -> None:
+        """
+        Abort the downloading of all files currently being transmitted.
+
+        :param project_name:
+        :return:
+        """
+        for file_name in dict(self.downloadingFiles):
+            messageJson = {"Operation": "File Download Abort",
+                           "Arguments": {
+                               'project_name': project_name,
+                               'file_name': file_name
+                           }}
+            self.transmitMessage(messageJson)
+            fileHandler = self.downloadingFiles.pop(file_name)
+            if fileHandler is None:
+                return
+            fileHandler.close()
+            abortedPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
+            abortedPath.unlink(missing_ok=True)

Core/GlobalVariables.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import random
+
+non_string_fields = ('Icon', 'Child UIDs')
+hidden_fields = ('uid', 'Date Last Edited', 'Child UIDs', 'Canvas Banner', 'Entity Type')
+hidden_fields_dockbars = ('uid', 'Child UIDs', 'Canvas Banner', 'Icon')
+meta_fields = ('Child UIDs',)
+avoid_parsing_fields = ('uid', 'Date Last Edited', 'Child UIDs', 'Icon', 'Canvas Banner')
+
+# Closer to the top means more recent.
+user_agents = {'Chrome': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                                      '(KHTML, like Gecko) Chrome/101.0.4951.15 Safari/537.36',),
+                          'Linux': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                                    'Chrome/101.0.4951.15 Safari/537.36',)
+                          },
+               'Firefox': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; rv:100.0) Gecko/20100101 Firefox/98.0',),
+                           'Linux': ('Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',)
+                           },
+               'Webkit': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 '
+                                      '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',),
+                          'Linux': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 '
+                                    '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',)
+                          }
+               }
+
+
+def getLatestUserAgent(browser: str = None, platform: str = None) -> str:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][0]
+
+
+def getUserAgents(browser: str = None, platform: str = None, amount: int = 1) -> tuple:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][:amount]

Core/Interface/DockBarOne.py

@@ -0,0 +1,426 @@
+#!/usr/bin/env python3
+from json import dumps
+from Core.Interface.Entity import BaseNode
+from PySide6 import QtWidgets, QtCore, QtGui
+
+
+class DockBarOne(QtWidgets.QDockWidget):
+
+    def initialiseLayout(self) -> None:
+        childWidget = QtWidgets.QTabWidget()
+        nodeTabChildWidget = QtWidgets.QWidget()
+        docsTabChildWidget = QtWidgets.QWidget()
+        nodeTabChildWidget.setLayout(QtWidgets.QGridLayout())
+        docsTabChildWidget.setLayout(QtWidgets.QGridLayout())
+        self.setWidget(childWidget)
+
+        childWidget.addTab(nodeTabChildWidget, "Node Operations")
+        childWidget.addTab(docsTabChildWidget, "Documents")
+
+        nodeTabChildWidget.layout().addWidget(self.nodesPalette)
+        nodeTabChildWidget.layout().addWidget(self.resolutionsPalette)
+        docsTabChildWidget.layout().addWidget(self.documentsList)
+        docsTabChildWidget.layout().addWidget(self.existingEntitiesPalette)
+
+        worldDocToggle = ToggleWorldDocButton(self, self.parent())
+        docsTabChildWidget.layout().addWidget(worldDocToggle)
+
+    def __init__(self,
+                 mainWindow,
+                 resolutionManager,
+                 resourceHandler,
+                 entityDatabase,
+                 title="DockBar One") -> None:
+        super(DockBarOne, self).__init__(parent=mainWindow)
+        self.resolutionManager = resolutionManager
+        self.resourceHandler = resourceHandler
+        self.lentDB = entityDatabase
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.LeftDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.setObjectName(title)
+
+        self.nodesPalette = NodeList(self.resourceHandler, self)
+        self.resolutionsPalette = ResolutionList(self.resolutionManager,
+                                                 self.lentDB,
+                                                 self.parent(),
+                                                 self)
+        self.documentsList = DocList(self.resourceHandler, self)
+        self.existingEntitiesPalette = EntityList(self.lentDB, self.parent(), self)
+
+        self.initialiseLayout()
+
+
+class ToggleWorldDocButton(QtWidgets.QPushButton):
+
+    def __init__(self, parent=None, mainWindow=None) -> None:
+        super(ToggleWorldDocButton, self).__init__(parent=parent)
+        self.mainWindow = mainWindow
+        self.setText("Toggle Detailed View")
+        self.clicked.connect(self.buttonPressed)
+
+    def buttonPressed(self) -> None:
+        if self.mainWindow is not None:
+            self.mainWindow.toggleWorldDoc()
+
+
+class EntityList(QtWidgets.QTreeWidget):
+
+    def __init__(self, entityDB, mainWindow, parent=None):
+        super(EntityList, self).__init__(parent=parent)
+
+        self.entityDB = entityDB
+        self.mainWindow = mainWindow
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Entity List'])
+        self.setAlternatingRowColors(False)
+        self.setMinimumWidth(200)
+        self.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection)
+        self.menu = QtWidgets.QMenu()
+
+        actionDelete = QtGui.QAction('Delete Selected Items',
+                                     self.menu,
+                                     statusTip="Delete the selected entities from the database.",
+                                     triggered=self.deleteSelectedItems)
+        self.menu.addAction(actionDelete)
+
+        actionAddToCurrentCanvas = QtGui.QAction('Add Selected Items to Current Canvas',
+                                                 self.menu,
+                                                 statusTip="Add the selected entities to the current canvas.",
+                                                 triggered=self.addItemsToCurrentCanvas)
+        self.menu.addAction(actionAddToCurrentCanvas)
+
+        self.entityCategories: dict = {}
+        self.entityTypes: dict = {}
+        self.loadEntities()
+
+    def loadEntities(self) -> None:
+        self.clear()
+        self.entityCategories = {}
+        self.entityTypes = {}
+
+        for category in self.mainWindow.RESOURCEHANDLER.getEntityCategories():
+            catTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            self.entityCategories[category] = catTreeItem
+
+        for category in self.entityCategories:
+            for entityType in self.mainWindow.RESOURCEHANDLER.getAllEntitiesInCategory(category):
+                typeTreeItem = QtWidgets.QTreeWidgetItem(self.entityCategories[category], [entityType])
+                self.entityTypes[entityType] = typeTreeItem
+
+        allEntities = self.entityDB.getAllEntities()
+        for entity in allEntities:
+            topItem = self.entityTypes[entity['Entity Type']]
+            primaryAttr = entity[list(entity)[1]]
+            pixmapIcon = QtGui.QPixmap()
+            pixmapIcon.loadFromData(entity['Icon'])
+            EntityWidget(topItem,
+                         entity['uid'],
+                         QtGui.QIcon(pixmapIcon),
+                         primaryAttr)
+
+        # Hide entity types and categories with no item instances (children):
+        for category in self.entityCategories:
+            childrenHidden = 0
+            for entityTypeIndex in range(self.entityCategories[category].childCount()):
+                entityTypeItem = self.entityCategories[category].child(entityTypeIndex)
+                if entityTypeItem.childCount() == 0:
+                    entityTypeItem.setHidden(True)
+                    childrenHidden += 1
+            if childrenHidden == self.entityCategories[category].childCount():
+                self.entityCategories[category].setHidden(True)
+
+    def addEntity(self, entityJson) -> None:
+        primaryAttr = entityJson[list(entityJson)[1]]
+        entityTypeItem = self.entityTypes[entityJson['Entity Type']]
+        for entityNo in range(entityTypeItem.childCount()):
+            child = entityTypeItem.child(entityNo)
+            if child.uid == entityJson['uid']:
+                child.setText(0, primaryAttr)
+                return
+        pixmapIcon = QtGui.QPixmap()
+        pixmapIcon.loadFromData(entityJson['Icon'])
+        EntityWidget(entityTypeItem,
+                     entityJson['uid'],
+                     QtGui.QIcon(pixmapIcon),
+                     primaryAttr)
+        # Un-hide parents of item, if they were hidden.
+        if entityTypeItem.isHidden():
+            entityTypeItem.setHidden(False)
+            entityTypeItem.parent().setHidden(False)
+
+    def removeEntity(self, entityJson) -> None:
+        entityType = self.entityTypes[entityJson['Entity Type']]
+        for entityNo in range(entityType.childCount()):
+            child = entityType.child(entityNo)
+            if child.uid == entityJson['uid']:
+                entityType.removeChild(child)
+                del child
+                if entityType.childCount() == 0:
+                    entityType.setHidden(True)
+                    entityCategory = entityType.parent()
+                    entityCategory.setHidden(True)
+                    for childID in range(entityCategory.childCount()):
+                        child = entityCategory.child(childID)
+                        if not child.isHidden():
+                            entityCategory.setHidden(False)
+                            break
+                break
+
+    def mouseMoveEvent(self, event) -> None:
+        """
+        Handle dragging of entities onto canvas.
+        """
+        super().mouseMoveEvent(event)
+
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
+            itemDragged = self.itemAt(event.pos())
+        else:
+            return
+
+        # Categories & entity names don't have uids.
+        try:
+            if itemDragged.uid is None:
+                return
+        except Exception:
+            return
+
+        self.setCurrentItem(itemDragged)
+        drag = QtGui.QDrag(self)
+        mimeData = QtCore.QMimeData()
+
+        mimeData.setText(dumps({'uid': itemDragged.uid}))
+        drag.setMimeData(mimeData)
+
+        # All Entities should have icons, but you never know.
+        pixmap = None
+        if itemDragged.icon(0) is not None:
+            pixmap = itemDragged.icon(0).pixmap(40, 40)
+            drag.setPixmap(pixmap)
+        drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() // 2, pixmap.rect().height() // 2))
+        drag.exec_()
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        super(EntityList, self).mousePressEvent(event)
+
+        if event.button() == QtGui.Qt.MouseButton.RightButton:
+            itemDragged = self.itemAt(event.pos())
+            if isinstance(itemDragged, EntityWidget):
+                self.menu.exec(QtGui.QCursor.pos())
+
+    def deleteSelectedItems(self) -> None:
+        itemsToDel = [item.uid for item in self.selectedItems() if isinstance(item, EntityWidget)]
+        for itemUID in itemsToDel:
+            self.mainWindow.deleteSpecificEntity(itemUID)
+
+    def addItemsToCurrentCanvas(self) -> None:
+        currentScene = self.mainWindow.centralWidget().tabbedPane.getCurrentScene()
+        itemsToAdd = [item.uid for item in self.selectedItems() if isinstance(item, EntityWidget) and
+                      item.uid not in currentScene.sceneGraph.nodes]
+        for itemUID in itemsToAdd:
+            if itemUID.endswith('@'):
+                newGroupEntity = self.mainWindow.copyGroupEntity(itemUID, currentScene)
+                if newGroupEntity is not None:
+                    currentScene.addNodeProgrammatic(newGroupEntity['uid'], newGroupEntity['Child UIDs'])
+            else:
+                currentScene.addNodeProgrammatic(itemUID)
+        currentScene.rearrangeGraph()
+
+
+class EntityWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, uid=None, icon=None, text="") -> None:
+        super(EntityWidget, self).__init__(parent, [text])
+        self.uid = uid
+        if icon is not None:
+            self.setIcon(0, icon)
+
+
+class DocList(QtWidgets.QTreeWidget):
+
+    def __init__(self, resourceHandler, parent=None) -> None:
+        super(DocList, self).__init__(parent=parent)
+
+        self.resourceHandler = resourceHandler
+        self.setAlternatingRowColors(False)
+        self.setHeaderLabels(['Files Loaded'])
+        self.uploadingFileWidgets = []
+        self.uploadedFileWidgets = []
+        self.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection)
+
+    def addUploadingFileToList(self, fileName: str) -> None:
+        newWidget = DocWidget(self,
+                              QtGui.QIcon(self.resourceHandler.getIcon("uploading")), fileName)
+        self.uploadingFileWidgets.append(newWidget)
+
+    def finishUploadingFile(self, fileName: str) -> None:
+        # Remove uploading file. Server will send updated file list.
+        for doc in list(self.uploadingFileWidgets):
+            if doc.getFileName() == fileName:
+                self.takeTopLevelItem(self.indexOfTopLevelItem(doc))
+                self.uploadingFileWidgets.remove(doc)
+                break
+
+    def updateFileListFromServer(self, fileList) -> None:
+        # If disconnected from server, or project has an empty file list, clear the widget.
+        if not fileList:
+            self.clear()
+            self.uploadingFileWidgets = []
+            self.uploadedFileWidgets = []
+            return
+
+        for fileName in fileList:
+            if fileName in [uploadedFileWidget.getFileName() for uploadedFileWidget in self.uploadedFileWidgets]:
+                continue
+            newWidget = DocWidget(self, None, fileName)
+            self.uploadedFileWidgets.append(newWidget)
+
+
+class DocWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, text="") -> None:
+        super(DocWidget, self).__init__(parent, [text])
+        if icon is not None:
+            self.setIcon(0, icon)
+
+    def getFileName(self) -> str:
+        return self.text(0)
+
+
+class ResolutionList(QtWidgets.QTreeWidget):
+
+    def __init__(self,
+                 resolutionManager,
+                 entityDatabase,
+                 mainWindow,
+                 parent=None) -> None:
+
+        super(ResolutionList, self).__init__(parent=parent)
+
+        self.resolutionManager = resolutionManager
+        self.lentDB = entityDatabase
+        self.mainWindow = mainWindow
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Resolutions'])
+        self.setAlternatingRowColors(False)
+        self.setMinimumWidth(200)
+        self.setSortingEnabled(True)
+        self.sortByColumn(0, QtCore.Qt.SortOrder.AscendingOrder)
+
+        self.loadAllResolutions()
+
+    def loadAllResolutions(self) -> None:
+        self.clear()
+        for category in self.resolutionManager.getResolutionCategories():
+            resTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            resolutions = self.resolutionManager.getResolutionsInCategory(category)
+            for res in resolutions:
+                ResolutionWidget(resTreeItem, text=res)
+
+    def loadResolutionsForSelected(self, selected) -> None:
+        self.clear()
+        if len(selected) == 0:
+            self.loadAllResolutions()
+            return
+        entityTypes = set()
+        for item in selected:
+            if not isinstance(item, BaseNode):
+                continue
+            ent = self.lentDB.getEntity(item.uid)
+            entityTypes.add(ent['Entity Type'])
+        res = self.resolutionManager.getResolutionsForEntityTypesByCategory(entityTypes)
+        for category in res:
+            resolutions = res[category]
+            if len(resolutions) != 0:
+                resTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+                for resolution in resolutions:
+                    ResolutionWidget(resTreeItem, text=resolution)
+
+    def mouseDoubleClickEvent(self, event) -> None:
+        super(ResolutionList, self).mouseDoubleClickEvent(event)
+        resItem = self.itemAt(event.pos())
+        if resItem is None or not isinstance(resItem, ResolutionWidget):
+            return
+        # Resolution Widgets should always have a parent, no need to check.
+        category = resItem.parent().text(0)
+        resolution = resItem.text(0)
+        if resolution not in self.resolutionManager.getResolutionsInCategory(category):
+            return
+
+        self.mainWindow.runResolution(f'{category}/{resolution}')
+
+
+class ResolutionWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, text="") -> None:
+        super(ResolutionWidget, self).__init__(parent, [text])
+        if icon is not None:
+            self.setIcon(0, icon)
+
+
+class NodeList(QtWidgets.QTreeWidget):
+
+    def __init__(self, resourceHandler, parent=None) -> None:
+        super(NodeList, self).__init__(parent=parent)
+
+        self.resourceHandler = resourceHandler
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Entities'])
+        self.setAlternatingRowColors(False)
+        self.setSortingEnabled(True)
+        self.sortByColumn(0, QtCore.Qt.SortOrder.AscendingOrder)
+        self.allEntities = []
+
+        self.loadEntities()
+
+    def loadEntities(self) -> None:
+        self.clear()
+        self.allEntities = []
+        for category in self.resourceHandler.getEntityCategories():
+            catTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            entities = self.resourceHandler.getAllEntityDetailsWithIconsInCategory(
+                category)
+            for entity in entities:
+                NodeWidget(catTreeItem,
+                           QtGui.QIcon(entity[1]),
+                           entity[0]['Entity Type'],
+                           dumps(entity[0]))
+
+        self.allEntities = self.resourceHandler.getAllEntities()
+
+    def mouseMoveEvent(self, event) -> None:
+        """
+        Handle dragging of entities onto canvas.
+        """
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
+            itemDragged = self.itemAt(event.pos())
+            if itemDragged is None or \
+                    itemDragged.text(0) not in self.allEntities:
+                return
+
+            self.setCurrentItem(itemDragged)
+            drag = QtGui.QDrag(self)
+            mimeData = QtCore.QMimeData()
+
+            mimeData.setText(itemDragged.entityJsonText)
+            drag.setMimeData(mimeData)
+
+            # All Entities should have icons, but you never know.
+            pixmap = None
+            if itemDragged.icon(0) is not None:
+                pixmap = itemDragged.icon(0).pixmap(40, 40)
+                drag.setPixmap(pixmap)
+            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() // 2, pixmap.rect().height() // 2))
+            drag.exec_()
+
+
+class NodeWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, name="", entityJsonText="") -> None:
+        super(NodeWidget, self).__init__(parent, [name])
+        if icon is not None:
+            self.setIcon(0, icon)
+        self.entityJsonText = entityJsonText

Core/Interface/DockBarThree.py

@@ -0,0 +1,521 @@
+#!/usr/bin/env python3
+
+import contextlib
+from PySide6 import QtWidgets, QtCore, QtCharts, QtGui
+from datetime import datetime
+from getpass import getuser
+import networkx as nx
+import queue
+
+
+class DockBarThree(QtWidgets.QDockWidget):
+    """
+    Dockbar that hosts the Timeline, Chat and Server Status widgets.
+    """
+
+    def initialiseLayout(self):
+        childWidget = QtWidgets.QWidget()
+        childWidget.setLayout(QtWidgets.QVBoxLayout())
+        childWidget.setContentsMargins(0, 0, 0, 0)
+        self.setWidget(childWidget)
+
+        childWidget2 = QtWidgets.QWidget()
+        childWidget2.setContentsMargins(0, 0, 0, 0)
+        childWidget2.setLayout(QtWidgets.QHBoxLayout())
+        childWidget.layout().addWidget(childWidget2)
+        childWidget2.layout().addWidget(self.tabPane)
+        self.tabPane.setContentsMargins(0, 0, 0, 0)
+        self.tabPane.addTab(self.logViewer, 'Program Log')
+        self.tabPane.addTab(self.timeWidget, 'Timeline')
+        childWidget2.layout().addWidget(self.chatBox)
+        childWidget.layout().addWidget(self.serverStatus)
+
+    def __init__(self, mainWindow, title="Dockbar Three"):
+        super(DockBarThree, self).__init__(parent=mainWindow)
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.TopDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.BottomDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.setObjectName(title)
+        self.setMinimumHeight(300)
+
+        self.tabPane = QtWidgets.QTabWidget()
+
+        self.serverStatus = ServerStatusBox(self)
+        self.chatBox = ChatBox(self, self.parent())
+        self.timeWidget = TimeWidget(self, self.parent())
+        self.logViewer = QtWidgets.QPlainTextEdit()
+
+        # Because we're not going to stop the thread before closing, an error will be thrown by Qt.
+        # That error can be safely ignored.
+        self.logViewerUpdateThread = LoggingUpdateThread(mainWindow.MESSAGEHANDLER)
+        self.logViewerUpdateThread.loggingSignal.connect(self.updateLogs)
+        self.logViewerUpdateThread.start()
+        self.logViewer.setReadOnly(True)
+
+        self.initialiseLayout()
+
+    def updateLogs(self, newLogMessage: str):
+        self.logViewer.appendPlainText(newLogMessage)
+
+
+class TimeWidget(QtWidgets.QWidget):
+
+    def __init__(self, parent, mainWindow):
+        super(TimeWidget, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.timeDetails = {}
+        self.currentTimeStep = []
+
+        self.timelineChart = QtCharts.QChart()
+        self.timelineChart.setTitle("Timeline")
+        self.timelineChart.setTheme(QtCharts.QChart.ChartTheme.ChartThemeBlueCerulean)
+        self.timelineChart.setMargins(QtCore.QMargins(0, 0, 0, 0))
+        self.chartView = QtCharts.QChartView(self.timelineChart)
+        self.chartView.setRubberBand(QtCharts.QChartView.RubberBand.NoRubberBand)
+        self.timelineChart.setAnimationOptions(QtCharts.QChart.AnimationOption.AllAnimations)
+        self.timelineChart.setAnimationDuration(250)
+        self.timelineChart.legend().hide()
+
+        self.timescaleSelector = TimelineTimescaleSelector(self)
+
+        self.setLayout(QtWidgets.QHBoxLayout())
+        self.layout().addWidget(self.timescaleSelector)
+        self.layout().addWidget(self.chartView)
+
+    def takePictureOfView(self, transparentBackground: bool = False):
+        # Need to set size and format of pic before using it.
+        # Ref: https://qtcentre.org/threads/10975-Help-Export-QGraphicsView-to-Image-File
+        # Rendering best optimized to rgb32 and argb32_premultiplied.
+        # Ref: https://doc.qt.io/qtforpython/PySide6/QtGui/QImage.html?highlight=qimage#image-formats
+        picture = QtGui.QImage(self.chartView.size(), QtGui.QImage.Format.Format_ARGB32_Premultiplied)
+        # Pictures are initialised with junk data - need to clear it out before painting
+        #   to avoid visual artifacts.
+        picture.fill(QtGui.QColor(0, 0, 0, 0))
+        picturePainter = QtGui.QPainter(picture)
+        if not transparentBackground:
+            picture.fill(QtGui.QColor(61, 61, 61))
+        self.chartView.render(picturePainter)
+        return picture
+
+    def updateTimeline(self, node, added: bool = True, updateGraph: bool = True):
+        nodeTime = datetime.fromisoformat(node['Date Created'])
+
+        nodeYear = nodeTime.year
+        nodeMonth = nodeTime.month
+        nodeDay = nodeTime.day
+        nodeHour = nodeTime.hour
+        nodeMinute = nodeTime.minute
+
+        # Dicts are fast, but this is not very efficient.
+        if nodeYear not in self.timeDetails:
+            self.timeDetails[nodeYear] = {}
+
+        if nodeMonth not in self.timeDetails[nodeYear]:
+            self.timeDetails[nodeYear][nodeMonth] = {}
+
+        if nodeDay not in self.timeDetails[nodeYear][nodeMonth]:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay] = {}
+
+        if nodeHour not in self.timeDetails[nodeYear][nodeMonth][nodeDay]:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour] = {}
+
+        if nodeMinute not in self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour]:
+            # Sanity check. Should not be able to remove nodes that do not exist, but you never know.
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 1 if added else 0
+
+        elif added:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] += 1
+        elif self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] > 0:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] -= 1
+
+        if updateGraph:
+            self.drawChart([])
+
+    def resetTimeline(self, graph: nx.DiGraph, updateGraph: bool = True):
+        self.timeDetails = {}
+
+        for node in graph.nodes():
+            self.updateTimeline(graph.nodes[node], True, False)
+
+        if updateGraph:
+            self.drawChart([])
+
+    def drawChart(self, timescale: list):
+        try:
+            year = timescale[0]
+        except IndexError:
+            year = None
+        try:
+            month = timescale[1]
+        except IndexError:
+            month = None
+        try:
+            day = timescale[2]
+        except IndexError:
+            day = None
+        try:
+            hour = timescale[3]
+        except IndexError:
+            hour = None
+        try:
+            minute = timescale[4]
+        except IndexError:
+            minute = None
+
+        if minute is not None:
+            barsDict = {minute: self.timeDetails[year][month][day][hour][minute]}
+            self.currentTimeStep = [year, month, day, hour, minute]
+        elif hour is not None:
+            barsDict = {minute: self.timeDetails[year][month][day][hour][minute]
+                        for minute in self.timeDetails[year][month][day][hour]}
+
+            self.currentTimeStep = [year, month, day, hour]
+        elif day is not None:
+            barsDict = {}
+            for hour in self.timeDetails[year][month][day]:
+                barsDict[hour] = 0
+                for minute in self.timeDetails[year][month][day][hour]:
+                    barsDict[hour] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year, month, day]
+        elif month is not None:
+            barsDict = {}
+            for day in self.timeDetails[year][month]:
+                barsDict[day] = 0
+                for hour in self.timeDetails[year][month][day]:
+                    for minute in self.timeDetails[year][month][day][hour]:
+                        barsDict[day] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year, month]
+        elif year is not None:
+            barsDict = {}
+            for month in self.timeDetails[year]:
+                barsDict[month] = 0
+                for day in self.timeDetails[year][month]:
+                    for hour in self.timeDetails[year][month][day]:
+                        for minute in self.timeDetails[year][month][day][hour]:
+                            barsDict[month] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year]
+        else:
+            barsDict = {}
+            for year in self.timeDetails:
+                barsDict[year] = 0
+                for month in self.timeDetails[year]:
+                    for day in self.timeDetails[year][month]:
+                        for hour in self.timeDetails[year][month][day]:
+                            for minute in self.timeDetails[year][month][day][hour]:
+                                barsDict[year] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = []
+
+        self.drawChartHelper(barsDict, self.currentTimeStep)
+
+    def drawChartHelper(self, barsDict: dict, timestep: list):
+        timelineSeries = QtCharts.QBarSeries(self.timelineChart)
+        maxEntityNum = 0
+        xAxisValues = []
+
+        barSet = TimelineBarSet('Entities', self, timestep, list(barsDict))
+        barSet.setColor(QtGui.Qt.GlobalColor.darkCyan)
+        for bar in barsDict:
+            barSet.append(barsDict[bar])
+            timelineSeries.append(barSet)
+            if barsDict[bar] > maxEntityNum:
+                maxEntityNum = barsDict[bar]
+            value = ""
+            for step, stepValue in enumerate(timestep):
+                if step <= 2:
+                    value += f'{str(stepValue)}/'
+                    if step == 2:
+                        value = f"{value[:-1]} "
+                else:
+                    value += f'{str(stepValue)}:'
+            value += str(bar)
+            xAxisValues.append(value)
+        self.timelineChart.removeAllSeries()
+        self.timelineChart.addSeries(timelineSeries)
+        self.timelineChart.createDefaultAxes()
+        self.timelineChart.removeAxis(self.timelineChart.axisX(timelineSeries))
+        self.timelineChart.removeAxis(self.timelineChart.axisY(timelineSeries))
+
+        yAxis = QtCharts.QValueAxis()
+        yAxis.setTickCount(min(maxEntityNum + 1, 4))
+
+        xAxis = QtCharts.QBarCategoryAxis()
+        xAxis.append(xAxisValues)
+        self.timelineChart.setAxisY(yAxis, timelineSeries)
+        self.timelineChart.setAxisX(xAxis, timelineSeries)
+
+        timesteps = len(timestep)
+        if timesteps == 0:
+            self.timescaleSelector.adjustLabelsToYear()
+        elif timesteps == 1:
+            self.timescaleSelector.adjustLabelsToMonth()
+        elif timesteps == 2:
+            self.timescaleSelector.adjustLabelsToDay()
+        elif timesteps == 3:
+            self.timescaleSelector.adjustLabelsToHour()
+        elif timesteps == 4:
+            self.timescaleSelector.adjustLabelsToMinute()
+        else:
+            self.timescaleSelector.adjustLabelsToSecond()
+
+        yAxis.applyNiceNumbers()
+        self.mainWindow.timelineSelectMatchingEntities(timestep)
+
+
+class TimelineBarSet(QtCharts.QBarSet):
+
+    def __init__(self, label: str, timeWidget: TimeWidget, timestep: list, barSeriesDict: list):
+        super(TimelineBarSet, self).__init__(label)
+
+        self.clicked.connect(self.selectedAction)
+        self.timeWidget = timeWidget
+        self.timestep = timestep
+        self.barSeriesDict = barSeriesDict
+
+    def selectedAction(self, index):
+        self.timestep.append(self.barSeriesDict[index])
+        self.timeWidget.drawChart(self.timestep)
+
+
+class TimelineTimescaleSelector(QtWidgets.QLabel):
+
+    def __init__(self, timeWidget: TimeWidget):
+        super(TimelineTimescaleSelector, self).__init__(parent=timeWidget)
+
+        self.timeWidget = timeWidget
+        self.setMinimumWidth(150)
+
+        self.setLayout(QtWidgets.QFormLayout())
+        self.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+
+        self.yearButton = QtWidgets.QPushButton(' Year: ')
+        self.yearButton.clicked.connect(self.yearButtonPressed)
+        self.yearText = QtWidgets.QLabel('-')
+        self.yearText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+        self.monthButton = QtWidgets.QPushButton(' Month: ')
+        self.monthButton.clicked.connect(self.monthButtonPressed)
+        self.monthText = QtWidgets.QLabel('X')
+        self.monthText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+        self.dayButton = QtWidgets.QPushButton(' Day: ')
+        self.dayButton.clicked.connect(self.dayButtonPressed)
+        self.dayText = QtWidgets.QLabel('X')
+        self.dayText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+        self.hourButton = QtWidgets.QPushButton(' Hour: ')
+        self.hourButton.clicked.connect(self.hourButtonPressed)
+        self.hourText = QtWidgets.QLabel('X')
+        self.hourText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+        self.minuteButton = QtWidgets.QPushButton(' Minute: ')
+        self.minuteButton.clicked.connect(self.minuteButtonPressed)
+        self.minuteText = QtWidgets.QLabel('X')
+        self.minuteText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
+
+        self.layout().addRow(self.yearButton, self.yearText)
+        self.layout().addRow(self.monthButton, self.monthText)
+        self.layout().addRow(self.dayButton, self.dayText)
+        self.layout().addRow(self.hourButton, self.hourText)
+        self.layout().addRow(self.minuteButton, self.minuteText)
+
+        self.yearButton.setDown(True)
+        self.yearButton.setDisabled(True)
+        self.monthButton.setDisabled(True)
+        self.dayButton.setDisabled(True)
+        self.hourButton.setDisabled(True)
+        self.minuteButton.setDisabled(True)
+
+    def yearButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:0])
+        self.adjustLabelsToYear()
+
+    def adjustLabelsToYear(self):
+        self.yearButton.setDown(True)
+        self.yearButton.setDisabled(True)
+        self.monthButton.setDisabled(True)
+        self.monthButton.setDown(False)
+        self.dayButton.setDisabled(True)
+        self.dayButton.setDown(False)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText('-')
+        self.monthText.setText('X')
+        self.dayText.setText('X')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def monthButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:1])
+        self.adjustLabelsToMonth()
+
+    def adjustLabelsToMonth(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(True)
+        self.monthButton.setDisabled(True)
+        self.dayButton.setDisabled(True)
+        self.dayButton.setDown(False)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText('-')
+        self.dayText.setText('X')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def dayButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:2])
+        self.adjustLabelsToDay()
+
+    def adjustLabelsToDay(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(True)
+        self.dayButton.setDisabled(True)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText('-')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def hourButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:3])
+        self.adjustLabelsToHour()
+
+    def adjustLabelsToHour(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(True)
+        self.hourButton.setDisabled(True)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText('-')
+        self.minuteText.setText('X')
+
+    def minuteButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:4])
+        self.adjustLabelsToMinute()
+
+    def adjustLabelsToMinute(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(False)
+        self.hourButton.setDisabled(False)
+        self.minuteButton.setDown(True)
+        self.minuteButton.setDisabled(True)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText(str(self.timeWidget.currentTimeStep[3]))
+        self.minuteText.setText('-')
+
+    def adjustLabelsToSecond(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(False)
+        self.hourButton.setDisabled(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(False)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText(str(self.timeWidget.currentTimeStep[3]))
+        self.minuteText.setText(str(self.timeWidget.currentTimeStep[4]))
+
+
+class ServerStatusBox(QtWidgets.QLabel):
+
+    def __init__(self, parent):
+        super(ServerStatusBox, self).__init__(parent=parent)
+        self.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken | QtWidgets.QFrame.Shape.StyledPanel)
+        self.setText("Not connected to a server")
+
+    def updateStatus(self, status: str):
+        self.setText(status)
+
+
+class ChatBox(QtWidgets.QWidget):
+    """
+    Sends chat messages to everyone connected to the same server.
+    """
+
+    def receiveMessage(self, message: str):
+        self.textView.appendPlainText(message)
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.setMinimumWidth(500)
+        self.setMaximumWidth(500)
+
+        self.chatName = f"{getuser()}: "
+
+        chatLayout = QtWidgets.QGridLayout()
+        self.setLayout(chatLayout)
+
+        chatLabel = QtWidgets.QLabel('Project Collaboration Chat')
+        chatLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.textView = QtWidgets.QPlainTextEdit()
+        self.textView.setReadOnly(True)
+        self.textSendBox = QtWidgets.QLineEdit()
+        self.textSendBox.setPlaceholderText("Type a message to send...")
+        self.textSendButton = QtWidgets.QPushButton(" Send ")
+        self.textSendButton.clicked.connect(self.sendMessage)
+
+        chatLayout.addWidget(chatLabel, 0, 0, 1, 2)
+        chatLayout.addWidget(self.textView, 1, 0, 1, 2)
+        chatLayout.addWidget(self.textSendBox, 2, 0)
+        chatLayout.addWidget(self.textSendButton, 2, 1)
+
+    def sendMessage(self):
+        self.mainWindow.sendChatMessage(self.chatName + self.textSendBox.text())
+        self.receiveMessage(self.chatName + self.textSendBox.text())
+        self.textSendBox.setText("")
+
+
+class LoggingUpdateThread(QtCore.QThread):
+    loggingSignal = QtCore.Signal(str)
+    endLogging = False
+
+    def __init__(self, messageHandler):
+        super().__init__()
+        self.messageHandler = messageHandler
+
+    def run(self):
+        while not self.endLogging:
+            if not self.messageHandler.logQueue.empty():
+                with contextlib.suppress(queue.Empty):
+                    logMsg = self.messageHandler.logQueue.get().getMessage()
+                    self.loggingSignal.emit(logMsg)
+            else:
+                self.msleep(100)

Core/Interface/DockBarTwo.py

@@ -0,0 +1,563 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import magic
+from PySide6 import QtWidgets, QtCore, QtGui
+from Core.ResourceHandler import MinSizeStackedLayout, RichNotesEditor
+from Core.GlobalVariables import hidden_fields_dockbars
+
+
+class DockBarTwo(QtWidgets.QDockWidget):
+
+    def initialiseLayout(self) -> None:
+        childWidget = QtWidgets.QTabWidget()
+        self.setWidget(childWidget)
+        scrollAreaWidget = QtWidgets.QScrollArea()
+
+        scrollAreaWidget.setWidget(self.entDetails)
+        scrollAreaWidget.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarPolicy.ScrollBarAsNeeded)
+        scrollAreaWidget.setWidgetResizable(True)
+        scrollAreaWidget.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
+
+        childWidget.addTab(scrollAreaWidget, 'Entity Details')
+        childWidget.addTab(self.oracle, 'Oracle')
+        childWidget.addTab(self.tabNotes, 'Notes')
+
+    def __init__(self,
+                 mainWindow,
+                 resourceHandler,
+                 entityDB,
+                 title="DockBar Two"):
+        super(DockBarTwo, self).__init__(parent=mainWindow)
+
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.LeftDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.resourceHandler = resourceHandler
+        self.entityDB = entityDB
+        self.setObjectName(title)
+
+        self.oracle = Oracle(self.parent(), self)
+        self.entDetails = EntityDetails(self.resourceHandler,
+                                        self.entityDB,
+                                        self.parent(),
+                                        self)
+        self.tabNotes = TabNotesPanel(self)
+
+        self.initialiseLayout()
+
+    def setNotesText(self, newText: str) -> None:
+        if not self.tabNotes.textEditor.isReadOnly():
+            self.tabNotes.textEditor.setReadOnly(True)
+        self.tabNotes.textEditor.contents = newText
+        self.tabNotes.textEditor.setMarkdown(newText)
+
+    def getNotesText(self) -> str:
+        """
+        Get text with the Markdown control characters.
+        @return:
+        """
+        return self.tabNotes.textEditor.toMarkdown()
+
+
+class TabNotesPanel(QtWidgets.QWidget):
+    """
+    This class is used to allow the user to take notes on a tab by tab
+    basis.
+    """
+
+    def __init__(self, parent=None):
+        super(TabNotesPanel, self).__init__(parent=parent)
+        tabNotesLayout = QtWidgets.QVBoxLayout()
+
+        self.textEditor = RichNotesEditor(self)
+
+        tabNotesLayout.addWidget(self.textEditor)
+        self.setLayout(tabNotesLayout)
+
+
+class EntityDetails(QtWidgets.QWidget):
+    """
+    This class is used to display a detailed view of the attributes of
+    the selected entity / entities.
+    """
+
+    def __init__(self,
+                 resourceHandler,
+                 entityDB,
+                 mainWindow,
+                 parent=None):
+
+        super(EntityDetails, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.resourceHandler = resourceHandler
+        self.entityDB = entityDB
+        self.detailsLayout = MinSizeStackedLayout()
+        self.setLayout(self.detailsLayout)
+        self.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
+
+        layoutNothingSelected = QtWidgets.QVBoxLayout()
+        widgetNothing = QtWidgets.QWidget()
+        widgetNothing.setLayout(layoutNothingSelected)
+        self.detailsLayout.addWidget(widgetNothing)
+
+        layoutOneNodeSelected = QtWidgets.QVBoxLayout()
+        widgetOneNode = QtWidgets.QWidget()
+        widgetOneNode.setLayout(layoutOneNodeSelected)
+        self.detailsLayout.addWidget(widgetOneNode)
+
+        layoutMultipleItemsSelected = QtWidgets.QVBoxLayout()
+        widgetMultiItems = QtWidgets.QWidget()
+        widgetMultiItems.setLayout(layoutMultipleItemsSelected)
+        self.detailsLayout.addWidget(widgetMultiItems)
+
+        # Need to keep track of how many nodes are selected.
+        # ~ Nothing Selected/Hovered Layout
+        nothingLabel = QtWidgets.QLabel("Nothing is Selected.")
+        nothingLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        layoutNothingSelected.addWidget(nothingLabel)
+        ###
+
+        # ~ One Link/Node Selected/Hovered Layout
+        # ~~ Part 1
+        summaryLayout = QtWidgets.QGridLayout()
+        summaryPanel = QtWidgets.QWidget()
+        summaryPanel.setLayout(summaryLayout)
+        self.summaryIcon = QtWidgets.QLabel("")
+        self.summaryIcon.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
+        self.entityTypeLabel = QtWidgets.QLabel("")
+        self.entityUIDLabel = QtWidgets.QLabel("")
+        self.entityPrimaryLabel = QtWidgets.QLabel("")
+        summaryLayout.addWidget(self.summaryIcon, 0, 0)
+        summaryLayout.addWidget(self.entityTypeLabel, 0, 1, 1, 3)
+        summaryLayout.addWidget(self.entityPrimaryLabel, 1, 1)
+        summaryLayout.addWidget(self.entityUIDLabel, 2, 1)
+        layoutOneNodeSelected.addWidget(summaryPanel, 1)
+
+        # ~~ Part 2
+        self.nodeLinkL = QtWidgets.QStackedLayout()
+        nodeLinkSwitcher = QtWidgets.QWidget()
+        nodeLinkSwitcher.setLayout(self.nodeLinkL)
+        layoutOneNodeSelected.addWidget(nodeLinkSwitcher, 1)
+
+        relationshipsLayout = QtWidgets.QVBoxLayout()
+        relationshipsPanel = QtWidgets.QWidget()
+        relationshipsPanel.setLayout(relationshipsLayout)
+        self.relationshipsIncomingTable = RelationshipsTable(self, mainWindow, uidLabel=self.entityPrimaryLabel,
+                                                             incomingOrOutgoing=0)
+        self.relationshipsOutgoingTable = RelationshipsTable(self, mainWindow, uidLabel=self.entityPrimaryLabel,
+                                                             incomingOrOutgoing=1)
+        self.relationshipsIncomingTable.setHeaderLabel('Incoming Links')
+        self.relationshipsOutgoingTable.setHeaderLabel('Outgoing Links')
+        relationshipsLayout.addWidget(self.relationshipsIncomingTable)
+        relationshipsLayout.addWidget(self.relationshipsOutgoingTable)
+        self.nodeLinkL.addWidget(relationshipsPanel)
+        self.relationshipsIncomingTable.setMinimumHeight(125)
+        self.relationshipsOutgoingTable.setMinimumHeight(125)
+
+        oneLinkRelLayout = QtWidgets.QHBoxLayout()
+        oneLinkRelPanel = QtWidgets.QWidget()
+        oneLinkRelPanel.setMaximumHeight(150)
+        oneLinkRelPanel.setLayout(oneLinkRelLayout)
+        self.linkParent = SingleLinkItem(self, mainWindow)
+        self.linkIcon = QtWidgets.QLabel("")
+        self.linkIcon.setMaximumHeight(90)
+        self.linkIcon.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.linkChild = SingleLinkItem(self, mainWindow)
+        oneLinkRelLayout.addWidget(self.linkParent)
+        oneLinkRelLayout.addWidget(self.linkIcon)
+        oneLinkRelLayout.addWidget(self.linkChild)
+        self.nodeLinkL.addWidget(oneLinkRelPanel)
+
+        # ~~ Part 3
+        scroll = QtWidgets.QScrollArea()
+
+        self.detailsLayoutOneNode = QtWidgets.QGridLayout()
+        detailsLayoutOneNodePanel = QtWidgets.QWidget()
+        detailsLayoutOneNodePanel.setLayout(self.detailsLayoutOneNode)
+        scroll.setWidget(detailsLayoutOneNodePanel)
+        scroll.setWidgetResizable(True)
+        scroll.setMinimumHeight(200)
+        layoutOneNodeSelected.addWidget(scroll, 10)
+
+        # ~ Multiple Links/Nodes Selected Layout
+        self.nodesTable = RelationshipsTable(self, mainWindow)
+        self.nodesTable.setMaximumHeight(500)
+        self.nodesTable.setHeaderLabels(["Entity", "Incoming Links", "Outgoing Links"])
+        self.linksTable = LinksTable(self, mainWindow)
+        self.linksTable.setMaximumHeight(500)
+        self.linksTable.setHeaderLabels(["Resolution", "Parent", "Child"])
+        self.multiNodesTableLabelOne = QtWidgets.QLabel("Selected Nodes:")
+        self.multiNodesTableLabelOne.setMaximumHeight(20)
+        layoutMultipleItemsSelected.addWidget(self.multiNodesTableLabelOne)
+        layoutMultipleItemsSelected.addWidget(self.nodesTable)
+        self.multiNodesTableLabelTwo = QtWidgets.QLabel("Selected Links:")
+        self.multiNodesTableLabelTwo.setMaximumHeight(20)
+        layoutMultipleItemsSelected.addWidget(self.multiNodesTableLabelTwo)
+        layoutMultipleItemsSelected.addWidget(self.linksTable)
+
+        self.currentlyShown = []
+
+    def displayWidgetDetails(self, jsonDicts) -> None:
+        """
+        jsonDicts: a list of json strings, each representing
+                   an entity or link.
+        isNode: Whether the json string(s) is/are nodes or links.
+        """
+        if jsonDicts == self.currentlyShown:
+            return
+        self.currentlyShown = jsonDicts
+        numberOfItems = len(jsonDicts)
+        if numberOfItems == 0:
+            self.detailsLayout.setCurrentIndex(0)
+            return
+        if None in jsonDicts:
+            self.mainWindow.MESSAGEHANDLER.error('Received None value when trying to display widget details.',
+                                                 popUp=False)
+            self.detailsLayout.setCurrentIndex(0)
+            return
+        self.clearDetailsHelper()
+        isNode = None
+        try:
+            if numberOfItems == 1:
+                entity = jsonDicts[0]
+                isNode = self.entityDB.isNode(entity['uid'])
+                self.populateOneSummaryHelper(entity, isNode)
+                self.populateOneDetailsHelper(entity)
+                if isNode:
+                    self.populateOneRelationshipHelper(entity['uid'])
+                else:
+                    self.populateOneLinkRelationshipHelper(entity['uid'])
+            else:
+                nodesNumber = 0
+                linksNumber = 0
+                for item in jsonDicts:
+                    if isinstance(item['uid'], str):
+                        self.populateMultiRelationshipHelperNode(item)
+                        nodesNumber += 1
+                    else:
+                        self.populateMultiRelationshipHelperLink(item)
+                        linksNumber += 1
+                self.multiNodesTableLabelOne.setText(f'Selected Nodes:   {str(nodesNumber)}')
+                self.multiNodesTableLabelTwo.setText(f'Selected Links:   {str(linksNumber)}')
+
+            self.switchLayoutHelper(numberOfItems, isNode)
+        except Exception as exc:
+            # If an error is thrown at some point during the process, show the default nothing selected screen.
+            self.detailsLayout.setCurrentIndex(0)
+            self.mainWindow.MESSAGEHANDLER.error(f'Error occurred while trying to display the details of the selected '
+                                                 f'nodes: {str(exc)}', popUp=False, exc_info=False)
+
+    # Display helper functions
+    def clearDetailsHelper(self) -> None:
+        """
+        Clear tree widget items from tree widgets.
+        """
+        itemToRemove = self.detailsLayoutOneNode.takeAt(0)
+        while itemToRemove is not None:
+            itemToRemove.widget().deleteLater()
+            del itemToRemove
+            itemToRemove = self.detailsLayoutOneNode.takeAt(0)
+
+        self.relationshipsIncomingTable.clear()
+        self.relationshipsOutgoingTable.clear()
+        self.nodesTable.clear()
+        self.linksTable.clear()
+
+    def populateOneDetailsHelper(self, jsonDict) -> None:
+        if jsonDict is None or jsonDict == []:
+            return
+        rowCount = 0
+        for key in jsonDict:
+            if key in hidden_fields_dockbars:
+                continue
+            elif key == "Notes":
+                notesTextArea = RichNotesEditor(self, jsonDict[key], False)
+                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
+                self.detailsLayoutOneNode.addWidget(notesTextArea, rowCount, 1, 10, 1)
+                rowCount += 9
+            else:
+                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
+                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(str(jsonDict[key])), rowCount, 1)
+            rowCount += 1
+        filePath = jsonDict.get('File Path')
+        if filePath is not None:
+            fullFilePath = Path(self.mainWindow.SETTINGS.value('Project/FilesDir')) / filePath
+            if fullFilePath.exists() and fullFilePath.is_file():
+                magicType = magic.from_file(str(fullFilePath), mime=True)
+                if magicType.split('/')[0] == 'image':
+                    previewImage = QtGui.QImage(fullFilePath)
+                    if previewImage.isNull():
+                        # If launched from terminal, expect errors like the following to show up:
+                        # qt.gui.imageio: QImageIOHandler: Rejecting image as it exceeds the current allocation
+                        #   limit of 128 megabytes
+                        # This is fine - we are handling this here, by having the label show text instead of a null
+                        #   image.
+                        previewLabel = QtWidgets.QLabel('Error Creating Image Preview.')
+                    else:
+                        previewPixmap = QtGui.QPixmap(previewImage)
+                        previewLabel = QtWidgets.QLabel()
+                        previewLabel.setPixmap(previewPixmap.scaled(250,
+                                                                    250,
+                                                                    QtCore.Qt.AspectRatioMode.KeepAspectRatio))
+                    self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel('Preview:'), rowCount, 0)
+                    self.detailsLayoutOneNode.addWidget(previewLabel, rowCount, 1, 10, 1)
+
+    def populateOneLinkRelationshipHelper(self, uid) -> None:
+        first = uid[0]
+        firstJson = self.entityDB.getEntity(first)
+        second = uid[1]
+        secondJson = self.entityDB.getEntity(second)
+        firstPixmap = QtGui.QPixmap()
+        firstPixmap.loadFromData(firstJson.get('Icon'))
+        secondPixmap = QtGui.QPixmap()
+        secondPixmap.loadFromData(secondJson.get('Icon'))
+        self.linkParent.linkItemPic.setPixmap(firstPixmap)
+        self.linkParent.linkItemName.setText(firstJson[list(firstJson)[1]])
+        self.linkParent.linkItemUid = firstJson['uid']
+        self.linkChild.linkItemPic.setPixmap(secondPixmap)
+        self.linkChild.linkItemName.setText(secondJson[list(secondJson)[1]])
+        self.linkChild.linkItemUid = secondJson['uid']
+
+    def populateOneRelationshipHelper(self, uid) -> None:
+        inc = self.entityDB.getIncomingLinks(uid)
+        out = self.entityDB.getOutgoingLinks(uid)
+        for edge in inc:
+            uid = edge[0]
+            edgeJson = self.entityDB.getEntity(uid)
+            nodePixmap = QtGui.QPixmap()
+            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            ResolutionTreeWidgetEntity(self.relationshipsIncomingTable,
+                                       nodePixmap,
+                                       edgeJson[list(edgeJson)[1]],
+                                       uid)
+        self.relationshipsIncomingTable.setHeaderLabel(f'Incoming Links:   {len(inc)}')
+        for edge in out:
+            uid = edge[1]
+            edgeJson = self.entityDB.getEntity(uid)
+            nodePixmap = QtGui.QPixmap()
+            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            ResolutionTreeWidgetEntity(self.relationshipsOutgoingTable,
+                                       nodePixmap,
+                                       edgeJson[list(edgeJson)[1]],
+                                       uid)
+        self.relationshipsOutgoingTable.setHeaderLabel(f'Outgoing Links:   {len(out)}')
+
+    def populateMultiRelationshipHelperNode(self, nodeJson) -> None:
+        inc = len(self.entityDB.getIncomingLinks(nodeJson['uid']))
+        out = len(self.entityDB.getOutgoingLinks(nodeJson['uid']))
+        nodePixmap = QtGui.QPixmap()
+        nodePixmap.loadFromData(nodeJson.get('Icon'))
+        ResolutionTreeWidgetEntity(self.nodesTable,
+                                   nodePixmap,
+                                   nodeJson[list(nodeJson)[1]],
+                                   nodeJson['uid'],
+                                   inc,
+                                   out)
+
+    def populateMultiRelationshipHelperLink(self, linkJson) -> None:
+        uid = linkJson['uid']
+        parent = self.entityDB.getEntity(uid[0])
+        child = self.entityDB.getEntity(uid[1])
+        LinksTreeWidgetEntity(self.linksTable,
+                              QtGui.QPixmap(self.resourceHandler.getLinkPicture()),
+                              linkJson['Resolution'],
+                              parent[list(parent)[1]],
+                              child[list(child)[1]],
+                              uid)
+
+    def populateOneSummaryHelper(self, jsonDict, isNode) -> None:
+        if isNode:
+            self.entityUIDLabel.setText(str(jsonDict[list(jsonDict)[1]]))
+            self.entityPrimaryLabel.setText(jsonDict[list(jsonDict)[0]])
+            self.entityTypeLabel.setText(jsonDict['Entity Type'])
+            summaryPixmap = QtGui.QPixmap()
+            summaryPixmap.loadFromData(jsonDict.get('Icon'))
+            self.summaryIcon.setPixmap(summaryPixmap)
+        else:
+            self.entityUIDLabel.setText("--")
+            self.entityPrimaryLabel.setText(jsonDict['Resolution'])
+            self.entityTypeLabel.setText('Resolution')
+            self.summaryIcon.setPixmap(QtGui.QPixmap(
+                self.resourceHandler.getLinkPicture()))
+            self.linkIcon.setPixmap(QtGui.QPixmap(
+                self.resourceHandler.getLinkArrowPicture()))
+
+    def switchLayoutHelper(self, selectionCount, isNode) -> None:
+        """
+        Switches between showing nothing, showing details for 1 node/link
+           and showing details for multiple nodes/links
+        Layout 0 = Nothing
+        Layout 1 = 1 node / link
+        Layout 2 = multi nodes / links
+        """
+        # selectionCount should never be 0, it is checked before this function is called
+        if selectionCount == 1:
+            if isNode:
+                self.detailsLayout.setCurrentIndex(1)
+                self.nodeLinkL.setCurrentIndex(0)
+            else:
+                self.detailsLayout.setCurrentIndex(1)
+                self.nodeLinkL.setCurrentIndex(1)
+        else:
+            self.detailsLayout.setCurrentIndex(2)
+
+
+class SingleLinkItem(QtWidgets.QWidget):
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+
+        self.linkItemPic = QtWidgets.QLabel()
+
+        self.linkItemPic.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.linkItemName = QtWidgets.QLabel()
+
+        self.linkItemName.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.linkItemUid = ""
+        self.setMaximumHeight(90)
+
+        linkItemLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(linkItemLayout)
+        linkItemLayout.addWidget(self.linkItemPic)
+        linkItemLayout.addWidget(self.linkItemName)
+        self.mainWindow = mainWindow
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        self.mainWindow.setCurrentCanvasSelection([self.linkItemUid])
+        super().mousePressEvent(event)
+
+
+class RelationshipsTable(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindow, uidLabel: QtWidgets.QLabel = None, incomingOrOutgoing: int = None):
+        super().__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.incomingOrOutgoing = incomingOrOutgoing
+        self.uidLabel = uidLabel
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        # The super() call is not technically needed.
+        super().mousePressEvent(event)
+        if event.button() == QtGui.Qt.MouseButton.LeftButton:
+            itemClicked = self.itemAt(event.pos())
+            if itemClicked is None:
+                return
+            if self.uidLabel is not None:
+                potentialSelectedEntityUID = self.uidLabel.text()
+                if potentialSelectedEntityUID == "":
+                    self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
+                elif self.incomingOrOutgoing is not None:
+                    if self.incomingOrOutgoing == 0:
+                        self.mainWindow.setCurrentCanvasSelection([(itemClicked.entityUID, potentialSelectedEntityUID)])
+                    else:
+                        self.mainWindow.setCurrentCanvasSelection([(potentialSelectedEntityUID, itemClicked.entityUID)])
+            else:
+                self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
+
+
+class LinksTable(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        if event.button() == QtGui.Qt.MouseButton.LeftButton:
+            itemClicked = self.itemAt(event.pos())
+            if itemClicked is None:
+                return
+            self.mainWindow.setCurrentCanvasSelection([itemClicked.uid])
+        else:
+            super().mousePressEvent(event)
+
+
+class LinksTreeWidgetEntity(QtWidgets.QTreeWidgetItem):
+    def __init__(self,
+                 parent,
+                 icon=None,
+                 resolutionText=None,
+                 ent1=None,
+                 ent2=None,
+                 uid=None):
+        super().__init__(parent)
+
+        self.setText(0, resolutionText)
+        self.setText(1, str(ent1))
+        self.setText(2, str(ent2))
+        self.uid = uid
+        self.setIcon(0, icon)
+        self.setIcon(1, icon)
+
+
+class ResolutionTreeWidgetEntity(QtWidgets.QTreeWidgetItem):
+    def __init__(self,
+                 parent,
+                 icon=None,
+                 name="",
+                 entityUID=None,
+                 relInc=None,
+                 relOut=None):
+
+        super().__init__(parent, [name])
+        if icon is not None:
+            self.setIcon(0, icon)
+        if relInc is not None:
+            self.setText(1, str(relInc))
+        if relOut is not None:
+            self.setText(2, str(relOut))
+        self.entityUID = entityUID
+
+
+class Oracle(QtWidgets.QWidget):
+    """
+    This class is used to query the Question Answering AI model.
+    It will be fed data from the files the user imports into
+      the software.
+    """
+
+    def __init__(self, mainWindow, parent=None):
+        super(Oracle, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        oracleLayout = QtWidgets.QGridLayout()
+        self.setLayout(oracleLayout)
+
+        self.answerLabel = QtWidgets.QLabel("Answer Section")
+        self.answerLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        oracleLayout.addWidget(self.answerLabel, 1, 0, 1, 2)
+        self.answerSection = QtWidgets.QPlainTextEdit()
+        self.answerSection.setReadOnly(True)
+        self.answerSection.setPlaceholderText(
+            "The answer to your Question will appear here.")
+        self.answerSection.setUndoRedoEnabled(False)
+        oracleLayout.addWidget(self.answerSection, 2, 0, 1, 2)
+
+        self.questionLabel = QtWidgets.QLabel("Ask a Question")
+        self.questionLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        oracleLayout.addWidget(self.questionLabel, 3, 0, 1, 2)
+        self.questionSection = QtWidgets.QLineEdit()
+        self.questionSection.setPlaceholderText("Ask a Question here.")
+
+        oracleLayout.addWidget(self.questionSection, 4, 0)
+
+        self.submitQuestionButton = QtWidgets.QPushButton("Ask")
+        oracleLayout.addWidget(self.submitQuestionButton, 4, 1)
+        self.questionSection.setMinimumWidth(200)
+
+        self.submitQuestionButton.clicked.connect(mainWindow.askQuestion)

Core/Interface/Entity.py

@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+
+import contextlib
+from json import dumps
+import math
+from typing import Any, Optional
+
+from PySide6 import QtCore, QtGui, QtWidgets
+from PySide6.QtWidgets import QGraphicsItem
+from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem, QGraphicsTextItem
+from PySide6.QtSvgWidgets import QGraphicsSvgItem
+
+ENTITY_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
+LINK_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
+
+
+class BaseNode(QGraphicsItemGroup):
+
+    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str, font: QtGui.QFont,
+                 brush: QtGui.QBrush) -> None:
+        super(BaseNode, self).__init__()
+
+        self.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+
+        if pictureByteArray.data().startswith(b'<svg '):
+            self.iconItem = QGraphicsSvgItem()
+            self.iconItem.renderer().load(pictureByteArray)
+            # Force recalculation of geometry, else this looks like 1 pixel.
+            # https://stackoverflow.com/a/68182093
+            self.iconItem.setElementId("")
+        else:
+            pixmapItem = QtGui.QPixmap()
+            pixmapItem.loadFromData(pictureByteArray)
+            self.iconItem = QGraphicsPixmapItem(pixmapItem)
+
+        self.labelItem = QGraphicsTextItem('')
+        # Have to do it this way; directly assigning stuff does not work due to how PySide6 works.
+        labelDocument = self.labelItem.document()
+        textOption = labelDocument.defaultTextOption()
+        textOption.setWrapMode(QtGui.QTextOption.WrapMode.WrapAtWordBoundaryOrAnywhere)
+        textOption.setAlignment(QtCore.Qt.AlignmentFlag.AlignHCenter)
+        labelDocument.setDefaultTextOption(textOption)
+        self.labelItem.setDocument(labelDocument)
+
+        self.bannerIconItem = QGraphicsSvgItem()
+        self.bannerIconItem.setElementId("")
+
+        self.addToGroup(self.iconItem)
+        self.addToGroup(self.labelItem)
+        self.addToGroup(self.bannerIconItem)
+
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(ENTITY_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setDefaultTextColor(brush.color())
+
+        self.updateLabel(primaryAttribute)
+
+        self.bannerIconItem.setPos(self.iconItem.x() + 15, self.iconItem.y() - 9)
+        self.bannerIconItem.setZValue(10)
+
+        self.uid = uid
+        self.setFlag(QGraphicsItem.ItemIsMovable, True)
+        self.setFlag(QGraphicsItem.ItemIsSelectable, True)
+        # Check if this option makes the software feel better or worse to use.
+        self.setFlag(QGraphicsItem.ItemClipsToShape, True)
+        self.setAcceptHoverEvents(True)
+
+        self.connectors = []
+
+    def updateLabel(self, newText: str = '') -> None:
+        if newText != '':
+            if len(newText) > 50:
+                newText = f"{newText[:47]}..."
+            self.labelItem.setPlainText(newText)
+        self.labelItem.document().adjustSize()
+        self.labelItem.setPos(self.iconItem.x() + 20 - (self.labelItem.textWidth() / 2), self.iconItem.y() + 45)
+
+    def updateBanner(self, bannerHidden: bool = True, bannerGraphic: QtCore.QByteArray = None) -> None:
+        if bannerHidden:  # No icon visible
+            self.bannerIconItem.hide()
+            self.bannerIconItem.setVisible(False)
+            return
+        self.bannerIconItem.renderer().load(bannerGraphic)
+        self.bannerIconItem.show()
+        self.bannerIconItem.setVisible(True)
+        self.bannerIconItem.setElementId("")
+
+    def removeConnector(self, connector) -> None:
+        # Exception could be thrown if the connector is already deleted.
+        with contextlib.suppress(ValueError):
+            self.connectors.remove(connector)
+
+    def addConnector(self, connector) -> None:
+        self.connectors.append(connector)
+
+    def getConnectorByUID(self, uid) -> None:
+        for connector in self.connectors:
+            if connector.uid == uid:
+                return connector
+
+    def hoverEnterEvent(self, event) -> None:
+        super().hoverEnterEvent(event)
+        self.scene().detailsWidgetCaller(uid=self.uid)
+
+    def hoverLeaveEvent(self, event) -> None:
+        super().hoverLeaveEvent(event)
+        self.scene().detailsWidgetCaller()
+
+    def mouseDoubleClickEvent(self, event) -> None:
+        self.scene().editEntityProperties(self.uid)
+        super().mouseDoubleClickEvent(event)
+
+    def shape(self) -> QtGui.QPainterPath:
+        returnPath = QtGui.QPainterPath()
+        returnPath.addRect(QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80))
+        return returnPath
+
+    def boundingRect(self) -> QtCore.QRectF:
+        return QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80)
+
+    def childrenBoundingRect(self) -> QtCore.QRectF:
+        return QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80)
+
+    def boundingRegion(self, itemToDeviceTransform: QtGui.QTransform) -> QtGui.QRegion:
+        return self.iconItem.boundingRegion(itemToDeviceTransform)
+
+    def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
+              widget: Optional[QtWidgets.QWidget] = ...) -> None:
+        painter.setPen(QtCore.Qt.PenStyle.NoPen)
+        if self.scene().views()[0].zoom < self.scene().hideZoom:
+            # Looks stupid, but fixes bug where entities are deselected when zooming out past hideZoom level.
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
+        else:
+            self.labelItem.show()
+        if self.isSelected():
+            centerPoint = QtCore.QPointF(self.iconItem.x() + 20, self.iconItem.y() + 20)
+            selectionBackgroundGradient = QtGui.QRadialGradient(centerPoint, 80, centerPoint)
+            selectionBackgroundGradient.setColorAt(0.0, QtGui.QColor(250, 250, 255))
+            selectionBackgroundGradient.setColorAt(0.5, QtGui.QColor(231, 240, 253, 0))
+            painter.setBrush(selectionBackgroundGradient)
+            painter.drawRect(QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80))
+        super(BaseNode, self).paint(painter, option, widget)
+
+
+class GroupNode(BaseNode):
+
+    # childNodes is a list of tuples, uid and picture, of all the nodes in the group.
+    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group', font=None, brush=None) -> None:
+        super(GroupNode, self).__init__(pictureByteArray, uid, label, font, brush)
+        self.groupedNodesConnectors = []
+        self.itemsThatWereGrouped = []
+        self.groupedNodesUid = set()
+
+        self.listWidget = GroupNodeChildList()
+        self.listProxyWidget = None
+
+    def itemChange(self, change: QtWidgets.QGraphicsItem.GraphicsItemChange, value: Any) -> Any:
+        if change == QtWidgets.QGraphicsItem.GraphicsItemChange.ItemSelectedChange:
+            if value:
+                self.showList(None)
+            else:
+                self.hideList()
+        return value
+
+    def showList(self, pos: QtCore.QPoint = None) -> None:
+        if self.listWidget is None or self.listProxyWidget is None:
+            return
+        self.listWidget.itemList.clear()
+        for uid in self.groupedNodesUid:
+            entityJson = self.scene().parent().entityDB.getEntity(uid)
+            try:
+                primaryField = entityJson[list(entityJson)[1]]
+            except IndexError:
+                primaryField = ''
+            iconPixmap = QtGui.QPixmap()
+            iconPixmap.loadFromData(entityJson['Icon'])
+
+            GroupNodeListItem(icon=iconPixmap, text=primaryField, uid=uid,
+                              listview=self.listWidget.itemList)
+        if pos is None:
+            pos = QtCore.QPoint(self.pos().x() + 60, self.pos().y())
+        self.listProxyWidget.setPos(pos)
+        self.listProxyWidget.setVisible(True)
+
+    def mouseMoveEvent(self, event: QtWidgets.QGraphicsSceneMouseEvent) -> None:
+        super(GroupNode, self).mouseMoveEvent(event)
+        if self.listProxyWidget is not None and self.listProxyWidget.isVisible():
+            self.listProxyWidget.setPos(QtCore.QPoint(self.pos().x() + 60, self.pos().y()))
+
+    def hideList(self) -> None:
+        if self.listProxyWidget is not None:
+            self.listProxyWidget.hide()
+            self.listProxyWidget.setVisible(False)
+
+    def formGroup(self, childNodeUIDs, listProxyWidget: QtWidgets.QGraphicsProxyWidget) -> None:
+        [self.addItemToGroup(uid) for uid in childNodeUIDs]  # Should be faster than just a for loop
+        self.listProxyWidget = listProxyWidget
+        self.listProxyWidget.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+
+    def addItemToGroup(self, uid: str) -> None:
+        self.groupedNodesUid.add(uid)
+        self.handleAddItemConnectors(uid)
+
+    def handleAddItemConnectors(self, uid) -> None:
+        incomingLinks = self.scene().parent().entityDB.getIncomingLinks(uid)
+        outgoingLinks = self.scene().parent().entityDB.getOutgoingLinks(uid)
+
+        # Suppress exceptions, as these give false alarms when initially drawing the graph on the canvas.
+        for link in incomingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
+        for link in outgoingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
+
+    def removeSpecificItemFromGroupIfExists(self, uid) -> bool:
+        if uid in self.groupedNodesUid:
+            self.groupedNodesUid.remove(uid)
+            self.setSelected(False)
+            return True
+        return False
+
+
+# Ref: https://doc.qt.io/qt-5/qtwidgets-graphicsview-diagramscene-example.html#arrow-class-definition
+# Ref: https://github.com/PySide/Examples/blob/master/examples/graphicsview/diagramscene/diagramscene.py
+class BaseConnector(QGraphicsItemGroup):
+
+    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None,
+                 font: QtGui.QFont = None, brush: QtGui.QBrush = None) -> None:
+        super(BaseConnector, self).__init__(parent)
+
+        self.myStartItem = origin
+        self.myEndItem = destination
+
+        self.labelItem = QGraphicsSimpleTextItem('')
+        self.addToGroup(self.labelItem)
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(LINK_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setBrush(brush)
+
+        self.updateLabel(name)
+
+        if uid is not None:
+            self.uid = set(uid) if isinstance(uid, (list, set)) else {uid}
+        else:
+            self.uid = {(origin.uid, destination.uid)}
+
+        self.setFlag(QGraphicsItem.ItemIsMovable, False)
+        self.setFlag(QGraphicsItem.ItemIsSelectable, True)
+        self.labelItem.setFlag(QGraphicsItem.ItemIsSelectable, False)
+
+        self.setZValue(-100)
+        self.myStartItem.addConnector(self)
+        self.myEndItem.addConnector(self)
+
+        self.colorSelected = QtGui.QColor(0, 173, 238)
+        self.colorDefault = QtGui.QColor(200, 200, 200)
+        self.myColor = self.colorDefault
+
+        self.pen = QtGui.QPen(self.myColor, 2, QtCore.Qt.PenStyle.SolidLine,
+                              QtCore.Qt.PenCapStyle.RoundCap, QtCore.Qt.PenJoinStyle.RoundJoin)
+
+        self.arrowHead = QtGui.QPolygonF()
+        self.line = QtCore.QLineF()
+
+    def updateLabel(self, newText: str = '') -> None:
+        if len(newText) > 50:
+            newText = f"{newText[:47]}..."
+        self.labelItem.setText(newText)
+        self.update()
+
+    def mousePressEvent(self, event) -> None:
+        self.scene().detailsWidgetCaller(uid=self.uid)
+        super().mousePressEvent(event)
+
+    def mouseDoubleClickEvent(self, event) -> None:
+        if len(self.uid) == 1:
+            self.scene().editLinkProperties(next(iter(self.uid)))
+        else:
+            # This will cause a message to be shown to the user, telling them that links representing multiple
+            #   connections cannot be edited.
+            self.scene().editLinkProperties("")
+        super().mouseDoubleClickEvent(event)
+
+    def startItem(self) -> BaseNode:
+        return self.myStartItem
+
+    def endItem(self) -> BaseNode:
+        return self.myEndItem
+
+    def updatePosition(self) -> None:
+        self.line = QtCore.QLineF(self.mapFromItem(self.myStartItem, 0, 0), self.mapFromItem(self.myEndItem, 0, 0))
+        self.update()
+
+    def boundingRect(self) -> QtCore.QRectF:
+        extra = self.pen.width() + 20
+        p1 = self.line.p1()
+        p2 = self.line.p2()
+        return QtCore.QRectF(p1, QtCore.QSizeF(p2.x() - p1.x(), p2.y() - p1.y())
+                             ).normalized().adjusted(-extra, -extra, extra, extra)
+
+    def shape(self) -> QtGui.QPainterPath:
+        path = QtGui.QPainterPath(self.line.p1())
+        path.lineTo(self.line.p2())
+        path.addPolygon(self.arrowHead)
+        return path
+
+    def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
+              widget: Optional[QtWidgets.QWidget] = ...) -> None:
+
+        currentStartPos = self.myStartItem.pos()
+        currentEndPos = self.myEndItem.pos()
+
+        self.myColor = self.colorSelected if self.isSelected() else self.colorDefault
+
+        p1 = QtCore.QPointF(currentStartPos.x() + 20, currentStartPos.y() + 20)
+        p2 = QtCore.QPointF(currentEndPos.x() + 20, currentEndPos.y() + 20)
+
+        line = QtCore.QLineF(p1, p2)
+
+        if line.length() < 45:
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
+            return
+
+        angle = math.atan2(line.dy(), - line.dx())
+
+        if (line.length() < 50 + len(self.labelItem.text()) * 15) or \
+                self.scene().views()[0].zoom < self.scene().hideZoom:
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
+        else:
+            self.labelItem.show()
+            angle2 = math.degrees(math.pi - angle)
+            if 90 < angle2 < 270:
+                angle2 -= 180
+                # When the label flips, it starts from 0.3 and reads towards the starting node (as opposed to
+                #   reading towards the ending node).
+                # To avoid overlap, we get the proportion of the text to the length of the line, and add it to the
+                #   offset of 0.3 to keep the label at the same relative location.
+                self.labelItem.setPos(line.pointAt(0.3 + (self.labelItem.boundingRect().width() / line.length())))
+            else:
+                self.labelItem.setPos(line.pointAt(0.3))
+            self.labelItem.setRotation(angle2)
+
+        myPen = QtGui.QPen(self.myColor)
+        arrowSize = 20.0
+        painter.setPen(myPen)
+        painter.setBrush(self.myColor)
+
+        line.setLength(line.length() - 45)
+        self.line = line
+
+        arrowP1 = line.p2() + QtCore.QPointF(
+            math.sin(angle + math.pi / 3.0) * arrowSize,
+            math.cos(angle + math.pi / 3.0) * arrowSize)
+        arrowP2 = line.p2() + QtCore.QPointF(
+            math.sin(angle + math.pi - math.pi / 3.0) * arrowSize,
+            math.cos(angle + math.pi - math.pi / 3.0) * arrowSize)
+
+        self.arrowHead.clear()
+        for point in [line.p2(), arrowP1, arrowP2]:
+            self.arrowHead.append(point)
+
+        painter.drawLine(line)
+        painter.drawPolygon(self.arrowHead)
+
+
+class GroupNodeListItem(QtWidgets.QListWidgetItem):
+
+    def __init__(self, icon, text, uid: str, listview=None) -> None:
+        super(GroupNodeListItem, self).__init__(icon, text, listview)
+        self.uid = uid
+
+
+class GroupNodeChildList(QtWidgets.QWidget):
+
+    def __init__(self) -> None:
+        super(GroupNodeChildList, self).__init__()
+
+        self.setLayout(QtWidgets.QVBoxLayout())
+        titleLabel = QtWidgets.QLabel('Child Items')
+        titleLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.layout().addWidget(titleLabel)
+
+        self.itemList = ChildListWidget()
+        self.layout().addWidget(self.itemList)
+
+
+class ChildListWidget(QtWidgets.QListWidget):
+
+    def __init__(self) -> None:
+        super(ChildListWidget, self).__init__()
+        self.setAlternatingRowColors(True)
+        self.setSortingEnabled(True)
+
+    def mouseMoveEvent(self, event: QtGui.QMouseEvent) -> None:
+        super().mouseMoveEvent(event)
+
+        itemDragged = None
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
+            itemDragged = self.itemAt(event.pos())
+
+        if itemDragged is None:
+            return
+
+        self.setCurrentItem(itemDragged)
+        drag = QtGui.QDrag(self)
+        mimeData = QtCore.QMimeData()
+
+        mimeData.setText(dumps({'uid': itemDragged.uid}))
+        drag.setMimeData(mimeData)
+
+        # All Entities should have icons, but you never know.
+        pixmap = None
+        if itemDragged.icon() is not None:
+            pixmap = itemDragged.icon().pixmap(100, 100)
+            drag.setPixmap(pixmap)
+        drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
+        drag.exec_()

Core/Interface/ToolBarOne.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+from PySide6 import QtWidgets, QtGui
+
+
+class ToolBarOne(QtWidgets.QToolBar):
+
+    def __init__(self, title, parent):
+        # Parent is (expected to be) mainWindow.
+        super().__init__(title, parent=parent)
+        self.setObjectName(title)
+        self.setToolButtonStyle(QtGui.Qt.ToolButtonStyle.ToolButtonTextUnderIcon)
+
+        newCanvas = QtGui.QAction("Add Canvas",
+                                  self,
+                                  statusTip="Create new Canvas or Open existing Canvas",
+                                  triggered=self.addCanvas,
+                                  icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('addCanvas')))
+        newCanvas.setShortcut('Ctrl+N')
+        self.addAction(newCanvas)
+        self.insertSeparator(newCanvas)
+
+        manualLink = QtGui.QAction('Create Manual &Link',
+                                   self,
+                                   statusTip="Link two entities together manually",
+                                   triggered=self.createManualLink,
+                                   icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('drawLink')))
+        self.addAction(manualLink)
+        self.insertSeparator(manualLink)
+
+        uploadSelected = QtGui.QAction('Upload Selected Files',
+                                       self,
+                                       statusTip="Upload the selected file entities to the server",
+                                       triggered=self.uploadSelectedFiles,
+                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('uploading')))
+        self.addAction(uploadSelected)
+        self.insertSeparator(uploadSelected)
+
+        selectLeaves = QtGui.QAction('Select Leaf Nodes',
+                                     self,
+                                     statusTip="Select nodes with one or more incoming links and no outgoing links.",
+                                     triggered=self.selectLeafNodes,
+                                     icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('leafNodes')))
+        self.addAction(selectLeaves)
+        self.insertSeparator(selectLeaves)
+
+        selectRoots = QtGui.QAction('Select Root Nodes',
+                                    self,
+                                    statusTip="Select nodes with one or more outgoing links and no incoming links.",
+                                    triggered=self.selectRootNodes,
+                                    icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('rootNodes')))
+        self.addAction(selectRoots)
+        self.insertSeparator(selectRoots)
+
+        selectIsolated = QtGui.QAction('Select Isolated Nodes',
+                                       self,
+                                       statusTip="Select nodes with no incoming or outgoing links.",
+                                       triggered=self.selectIsolatedNodes,
+                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('isolatedNodes')))
+        self.addAction(selectIsolated)
+        self.insertSeparator(selectIsolated)
+
+        selectNonIsolated = QtGui.QAction('Select Non Isolated Nodes',
+                                          self,
+                                          statusTip="Select nodes with at least one incoming or outgoing link.",
+                                          triggered=self.selectNonIsolatedNodes,
+                                          icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('nonIsolatedNodes')))
+        self.addAction(selectNonIsolated)
+        self.insertSeparator(selectNonIsolated)
+
+        shortestPath = QtGui.QAction('Find Shortest Path',
+                                     self,
+                                     statusTip="Find the shortest path (if any) between two selected entities.",
+                                     triggered=self.findShortestPath,
+                                     icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('shortestPath')))
+        self.addAction(shortestPath)
+        self.insertSeparator(shortestPath)
+        mergeEntities = QtGui.QAction('Merge Entities',
+                                      self,
+                                      statusTip="Merge the selected entities into one.",
+                                      triggered=self.mergeEntities,
+                                      icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('merge')))
+        self.addAction(mergeEntities)
+        self.insertSeparator(mergeEntities)
+
+        splitEntity = QtGui.QAction('Split Entity',
+                                    self,
+                                    statusTip="Split the selected entity into multiple different ones.",
+                                    triggered=self.splitEntity,
+                                    icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('split')))
+        self.addAction(splitEntity)
+        self.insertSeparator(splitEntity)
+
+        generateReportAction = QtGui.QAction('Generate Report',
+                                             self,
+                                             statusTip="Generate Report from the selected nodes.",
+                                             triggered=self.generateReport,
+                                             icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
+        self.addAction(generateReportAction)
+        self.insertSeparator(generateReportAction)
+
+        rearrangeCanvas = QtGui.QAction('Rearrange Canvas',
+                                        self,
+                                        statusTip="Rearrange Canvas Nodes automatically.",
+                                        triggered=self.rearrangeGraph,
+                                        icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('rearrange')))
+        self.addAction(rearrangeCanvas)
+        self.insertSeparator(rearrangeCanvas)
+
+        self.linkItems = []
+        self.selectMode = False
+
+    def createManualLink(self):
+        self.parent().toggleLinkingMode()
+
+    def rearrangeGraph(self):
+        self.parent().centralWidget().tabbedPane.getCurrentScene().rearrangeGraph()
+
+    def addCanvas(self):
+        self.parent().addCanvas()
+
+    def selectLeafNodes(self):
+        self.parent().selectLeafNodes()
+
+    def selectRootNodes(self):
+        self.parent().selectRootNodes()
+
+    def selectIsolatedNodes(self):
+        self.parent().selectIsolatedNodes()
+
+    def selectNonIsolatedNodes(self):
+        self.parent().selectNonIsolatedNodes()
+
+    def findShortestPath(self):
+        self.parent().findShortestPath()
+
+    def uploadSelectedFiles(self):
+        self.parent().uploadFiles()
+
+    def downloadSelectedFiles(self):
+        self.parent().downloadFiles()
+
+    def mergeEntities(self):
+        self.parent().mergeEntities()
+
+    def splitEntity(self):
+        self.parent().splitEntity()
+
+    def generateReport(self):
+        self.parent().generateReport()

Core/LQL.py

@@ -0,0 +1,433 @@
+#!/usr/bin/env python3
+
+import contextlib
+from typing import Union, Optional, Any
+from uuid import uuid4
+import re
+import networkx as nx
+import string
+
+from Core.GlobalVariables import non_string_fields
+
+"""
+This class handles the backend stuff for the LinkScope Query Language.
+"""
+
+
+class LQLQueryBuilder:
+    QUERIES_HISTORY = {}
+
+    databaseSnapshot = None
+    databaseEntities = None
+    allCanvases = None
+    canvasesEntitiesDict = None
+    allEntityFields = None
+    allEntities = None
+
+    def __init__(self, mainWindow):
+        self.mainWindow = mainWindow
+
+    def takeSnapshot(self):
+        with self.mainWindow.LENTDB.dbLock:
+            # Create a copy
+            self.databaseSnapshot = self.mainWindow.LENTDB.database.copy()
+
+        self.databaseEntities = set(self.databaseSnapshot.nodes)
+
+        self.allCanvases = self.getAllCanvasNames()
+        self.canvasesEntitiesDict = self.getCanvasesEntitiesDict(self.allCanvases)
+        self.allEntityFields, self.allEntities = self.getAllEntitiesAndFields()
+
+        # Re-define database entities to remove Group Entities
+        self.databaseEntities = set(self.allEntities.keys())
+
+    def getAllEntitiesAndFields(self) -> (set, list):
+        entitiesSnapshot = {entity: self.databaseSnapshot.nodes[entity] for entity in self.databaseSnapshot.nodes
+                            if self.databaseSnapshot.nodes[entity].get('Entity Type') != 'EntityGroup'}
+        entityFields = set()
+        for entityUID in entitiesSnapshot:
+            entityFields.update(entitiesSnapshot[entityUID].keys())
+        for field in non_string_fields:
+            try:
+                entityFields.remove(field)
+            except KeyError:
+                # This typically only happens if there are no entities in the database.
+                continue
+        return entityFields, entitiesSnapshot
+
+    def getAllCanvasNames(self) -> list:
+        canvasNames = list(self.mainWindow.centralWidget().tabbedPane.canvasTabs.keys())
+        canvasNames.append('*')
+        return canvasNames
+
+    def getEntitiesOnCanvas(self, canvasName: str):
+        try:
+            # Ensure that we don't have nodes here that are not present in our database snapshot
+            canvasNodes = set(
+                self.mainWindow.centralWidget().tabbedPane.canvasTabs[canvasName].scene().sceneGraph.nodes)
+            return canvasNodes.intersection(self.databaseEntities)
+        except KeyError:
+            return None
+
+    def getCanvasesEntitiesDict(self, allCanvasNames: list):
+        returnDict = {}
+        for canvas in allCanvasNames:
+            allEntitiesOnCanvas = self.getEntitiesOnCanvas(canvas)
+            if allEntitiesOnCanvas is not None:
+                returnDict[canvas] = allEntitiesOnCanvas
+            else:
+                returnDict[canvas] = set()
+
+        return returnDict
+
+    def parseSelect(self, selectClause: str, selectValue: Union[str, list]):
+        if selectClause == 'SELECT':
+            return self.allEntityFields if '*' in selectValue else \
+                {entityField for entityField in selectValue if entityField in self.allEntityFields}
+
+        try:
+            clauseValue = re.compile(selectValue)
+            return {entityField for entityField in self.allEntityFields if clauseValue.match(entityField)}
+        except re.error:
+            return set()
+
+    def parseSource(self, sourceClause: str, sourceValues: Union[None, list], fieldsToSelect: set) -> set:
+        """
+        sourceValues:
+        [[("AND" | "OR" | None), ("CANVAS" | "RCANVAS"), (True | False), <User Input>], ...]
+        OR
+        None
+            if sourceClause == "FROMDB"
+        """
+        if sourceClause == "FROMDB":
+            resultEntitySet = set(self.databaseEntities)
+        else:
+            resultEntitySet = set()
+            for sourceValue in sourceValues:
+                try:
+                    if sourceValue[1] == "CANVAS":
+                        if sourceValue[3] not in self.allCanvases:
+                            raise ValueError('Reference to nonexistent canvas.')
+                        matchingCanvases = [sourceValue[3]]
+                    else:
+                        canvasRegex = re.compile(sourceValue[3])
+                        matchingCanvases = [canvasMatch for canvasMatch in self.allCanvases
+                                            if canvasRegex.match(canvasMatch)]
+                except (ValueError, re.error):
+                    continue
+
+                for matchingCanvas in matchingCanvases:
+                    if sourceValue[0] == 'AND':
+                        resultEntitySet = self.canvasAndNot(resultEntitySet, self.canvasesEntitiesDict[matchingCanvas])\
+                            if sourceValue[2] is True else\
+                            self.canvasAnd(resultEntitySet, self.canvasesEntitiesDict[matchingCanvas])
+
+                    elif sourceValue[2] is True:
+                        resultEntitySet = self.canvasOrNot(resultEntitySet,
+                                                           self.canvasesEntitiesDict[matchingCanvas],
+                                                           self.databaseEntities)
+                    else:
+                        resultEntitySet = self.canvasOr(resultEntitySet,
+                                                        self.canvasesEntitiesDict[matchingCanvas])
+
+        # Filter out all entities that do not contain at least one of the selected fields.
+        for entity in list(resultEntitySet):
+            validEntity = any(field in self.allEntities[entity].keys() for field in fieldsToSelect)
+
+            if not validEntity:
+                resultEntitySet.remove(entity)
+                self.allEntities.pop(entity)
+        return resultEntitySet
+
+    def parseConditions(self, conditionClauses: Union[None, list], entitiesPool) -> set:
+        """
+        conditionClauses:
+        [[("AND" | "OR" | None), ("Value Condition" | "Graph Condition"), (True | False), conditionValue], ...]
+
+        conditionValue:
+            if Value Condition:
+                [("ATTRIBUTE" | "RATTRIBUTE"), <User Input>,
+                ("EQ" | "CONTAINS" | "STARTSWITH" | "ENDSWITH" | "RMATCH"), <User Input>]
+            if Graph Condition:
+                [("CHILDOF" <ENTITY> | "DESCENDANTOF " <ENTITY> |
+                "PARENTOF" <ENTITY> | "ANCESTOROF " <ENTITY> |
+                "NUMCHILDREN" (" < " | " <= " | " > " | " >= " | " == ") <DIGITS> |
+                "NUMPARENTS" (" < " | " <= " | " > " | " >= " | " == ") <DIGITS> |
+                "CONNECTEDTO" <ENTITY> | "ISOLATED" | "ISROOT" | "ISLEAF")]
+        """
+
+        self.allEntities = {uid: self.allEntities[uid] for uid in self.allEntities if uid in entitiesPool}
+        uidsToSelect = set()
+
+        for conditionClause in conditionClauses:
+            isNot = conditionClause[2]
+            conditionValue = conditionClause[3]
+            try:
+                userInput1 = conditionValue[1]
+                userInput2 = conditionValue[3]
+            except IndexError:
+                # Not used in cases where a Graph Condition is specified
+                userInput1 = None
+                userInput2 = None
+            firstArgument = conditionValue[0]
+            if conditionClause[1] == "Value Condition":
+                matchingFields = []
+                if firstArgument == "ATTRIBUTE":
+                    if userInput1 in self.allEntityFields:
+                        matchingFields.append(userInput1)
+                else:
+                    try:
+                        attributeRegex = re.compile(userInput1)
+                    except re.error:
+                        continue
+                    matchingFields.extend(field for field in self.allEntityFields if attributeRegex.match(field))
+
+                for matchingField in matchingFields:
+                    entitiesToRemove = []
+                    for entity in self.allEntities:
+                        attributeKeyValue = str(self.allEntities[entity].get(matchingField))
+                        if self.checkVCHelper(conditionValue[2], isNot, attributeKeyValue, userInput2):
+                            uidsToSelect.add(entity)
+                        elif conditionClause[0] == "AND":
+                            entitiesToRemove.append(entity)
+                    for entityToRemove in entitiesToRemove:
+                        uidsToSelect.remove(entityToRemove)
+
+            elif conditionClause[1] == "Graph Condition":
+                entitiesToRemove = []
+                for entity in self.allEntities:
+                    if self.checkGCHelper(firstArgument, isNot, [entity] + conditionValue[1:]):
+                        uidsToSelect.add(entity)
+                    elif conditionClause[0] == "AND":
+                        entitiesToRemove.append(entity)
+                for entityToRemove in entitiesToRemove:
+                    uidsToSelect.remove(entityToRemove)
+
+        uidsToRemove = set(self.allEntities).difference(uidsToSelect)
+        for entity in uidsToRemove:
+            self.allEntities.pop(entity, None)
+
+        return uidsToSelect
+
+    def canvasOr(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.union(canvasSetB)
+
+    def canvasAnd(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.intersection(canvasSetB)
+
+    def canvasAndNot(self, canvasSetA: set, canvasSetB: set):
+        return canvasSetA.difference(canvasSetB)
+
+    def canvasOrNot(self, canvasSetA: set, canvasSetB: set, allEntitiesSet: set):
+        return canvasSetA.union(allEntitiesSet.difference(canvasSetB))
+
+    def checkEQ(self, valueA: str, valueB: str):
+        return valueA == valueB
+
+    def checkContains(self, valueA: str, valueB: str):
+        return valueB in valueA
+
+    def checkStartsWith(self, valueA: str, valueB: str):
+        return valueA.startswith(valueB)
+
+    def checkEndsWith(self, valueA: str, valueB: str):
+        return valueA.endswith(valueB)
+
+    def checkRMatch(self, valueA: str, valueB: str):
+        with contextlib.suppress(re.error):
+            valueMatch = re.compile(valueB)
+            if valueMatch.match(valueA):
+                return True
+        return False
+
+    def checkVCHelper(self, checkType: str, isNot: bool, valueA: str, valueB: str):
+        returnVal = False
+        if checkType == "CONTAINS":
+            returnVal = self.checkContains(valueA, valueB)
+        elif checkType == "ENDSWITH":
+            returnVal = self.checkEndsWith(valueA, valueB)
+        elif checkType == "EQ":
+            returnVal = self.checkEQ(valueA, valueB)
+        elif checkType == "RMATCH":
+            returnVal = self.checkRMatch(valueA, valueB)
+        elif checkType == "STARTSWITH":
+            returnVal = self.checkStartsWith(valueA, valueB)
+        return not returnVal if isNot else returnVal
+
+    def checkParentOf(self, valueA: str, valueB: str):
+        return self.databaseSnapshot.has_successor(valueA, valueB)
+
+    def checkAncestorOf(self, valueA: str, valueB: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if valueB in nx.descendants(self.databaseSnapshot, valueA):
+                return True
+        return False
+
+    def checkChildOf(self, valueA: str, valueB: str):
+        return self.databaseSnapshot.has_predecessor(valueA, valueB)
+
+    def checkDescendantOf(self, valueA: str, valueB: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if valueB in nx.ancestors(self.databaseSnapshot, valueA):
+                return True
+        return False
+
+    def checkNumChildren(self, valueA: str, valueB: str, valueC: int):
+        numChildren = len(list(self.databaseSnapshot.successors(valueA)))
+        return (valueB == "<" and numChildren < valueC) or \
+               (valueB == "<=" and numChildren <= valueC) or \
+               (valueB == ">" and numChildren > valueC) or \
+               (valueB == ">=" and numChildren >= valueC) or \
+               (valueB == "==" and numChildren == valueC)
+
+    def checkNumParents(self, valueA: str, valueB: str, valueC: int):
+        numParents = len(list(self.databaseSnapshot.predecessors(valueA)))
+        return (valueB == "<" and numParents < valueC) or \
+               (valueB == "<=" and numParents <= valueC) or \
+               (valueB == ">" and numParents > valueC) or \
+               (valueB == ">=" and numParents >= valueC) or \
+               (valueB == "==" and numParents == valueC)
+
+    def checkConnectedTo(self, valueA: str, valueB: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if nx.has_path(self.databaseSnapshot, valueA, valueB):
+                return True
+        return False
+
+    def checkIsolated(self, valueA: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if valueA in self.databaseSnapshot.nodes and nx.is_isolate(self.databaseSnapshot, valueA):
+                return True
+        return False
+
+    def checkIsRoot(self, valueA: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if len(self.databaseSnapshot.in_edges(valueA)) == 0:
+                return True
+        return False
+
+    def checkIsLeaf(self, valueA: str):
+        with contextlib.suppress(nx.NetworkXError):
+            if len(self.databaseSnapshot.out_edges(valueA)) == 0:
+                return True
+        return False
+
+    def checkGCHelper(self, checkType: str, isNot: bool, args: list):
+        returnVal = False
+        if checkType == "ANCESTOROF":
+            returnVal = self.checkAncestorOf(*args)
+        elif checkType == "CHILDOF":
+            returnVal = self.checkChildOf(*args)
+        elif checkType == "CONNECTEDTO":
+            returnVal = self.checkConnectedTo(*args)
+        elif checkType == "DESCENDANTOF":
+            returnVal = self.checkDescendantOf(*args)
+        elif checkType == "ISLEAF":
+            returnVal = self.checkIsLeaf(*args)
+        elif checkType == "ISOLATED":
+            returnVal = self.checkIsolated(*args)
+        elif checkType == "ISROOT":
+            returnVal = self.checkIsRoot(*args)
+        elif checkType == "NUMCHILDREN":
+            returnVal = self.checkNumChildren(*args)
+        elif checkType == "NUMPARENTS":
+            returnVal = self.checkNumParents(*args)
+        elif checkType == "PARENTOF":
+            returnVal = self.checkParentOf(*args)
+        return not returnVal if isNot else returnVal
+
+    def modifyNumify(self, valueA: str):
+        # Get the first number that shows up.
+        tempString = valueA.replace(',', '.')  # Making sure that floats are expressed the right way.
+        count = 0
+        for c in tempString:
+            if c not in string.digits:
+                count += 1
+            else:
+                break
+
+        count2 = 0
+        for c in tempString[count:]:
+            if c in string.digits or c == '.':
+                count2 += 1
+            else:
+                break
+
+        # If there are no numbers in the string, its numeric value is 0.
+        try:
+            floatValue = float(tempString[count:count + count2])
+        except ValueError:
+            floatValue = 0.0
+        return str(floatValue)
+
+    def modifyUpperCase(self, valueA: str):
+        return valueA.upper()
+
+    def modifyLowerCase(self, valueA: str):
+        return valueA.lower()
+
+    def parseModify(self, resultsToModify: (set, set), modifyQueries: list) -> (set, set):
+        """
+        modifyQueries:
+        [[("MODIFY" | "RMODIFY"), <User Input>, ("NUMIFY" | "UPPERCASE" | "LOWERCASE")], ...]
+        """
+
+        matchingFields = resultsToModify[1]
+
+        modifiedUIDs = set()
+        numifiedFields = set()
+
+        for modification in modifyQueries:
+            userInput1 = modification[1]
+            modificationType = modification[2]
+            modifyFields = []
+            if modification[0] == "MODIFY":
+                if userInput1 in resultsToModify[1]:
+                    modifyFields.append(userInput1)
+            else:
+                try:
+                    userInputRegex = re.compile(userInput1)
+                    modifyFields = [fieldMatch for fieldMatch in matchingFields if userInputRegex.match(fieldMatch)]
+                except (ValueError, re.error):
+                    continue
+            for entity in self.allEntities:
+                for modifyField in modifyFields:
+                    entityFieldValue = self.allEntities[entity].get(modifyField)
+                    if entityFieldValue is None or modificationType not in ["UPPERCASE", "LOWERCASE", "NUMIFY"]:
+                        newFieldValue = None
+                    elif modificationType == "UPPERCASE":
+                        newFieldValue = self.modifyUpperCase(entityFieldValue)
+                    elif modificationType == "LOWERCASE":
+                        newFieldValue = self.modifyLowerCase(entityFieldValue)
+                    else:
+                        newFieldValue = self.modifyNumify(entityFieldValue)
+                        numifiedFields.add(modifyField)
+                    if newFieldValue is not None:
+                        modifiedUIDs.add(entity)
+                        self.allEntities[entity][modifyField] = newFieldValue
+
+        return modifiedUIDs, numifiedFields
+
+    def parseQuery(self, selectClause: str, selectValue: Union[str, list], sourceClause: str,
+                   sourceValues: Union[None, list], conditionClauses: Union[None, list],
+                   modifyQueries: Union[list, None] = None) -> Optional[tuple[Optional[tuple[set, Union[set[Any], set[Union[str, Any]]]]],
+                                                                              Optional[tuple[set[Any], set[Any]]]]]:
+
+        if self.databaseSnapshot is None:
+            return None
+
+        returnValue = None
+        modifications = None
+        if fieldsToSelect := self.parseSelect(selectClause, selectValue):
+            if entitiesToConsider := self.parseSource(sourceClause, sourceValues, fieldsToSelect):
+                if conditionClauses:
+                    entitiesToConsider = self.parseConditions(conditionClauses, entitiesToConsider)
+                returnValue = (entitiesToConsider, fieldsToSelect)
+                if modifyQueries:
+                    modifications = self.parseModify(returnValue, modifyQueries)
+
+        queryUID = str(uuid4())
+        self.QUERIES_HISTORY[queryUID] = (selectClause, selectValue, sourceClause, sourceValues, conditionClauses,
+                                          modifyQueries)
+
+        return returnValue, modifications

Core/MessageHandler.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import logging
+from logging import handlers
+
+from multiprocessing import Queue
+from pathlib import Path
+from PySide6 import QtWidgets
+
+
+class MessageHandler:
+    """
+    Class that handles logging.
+    Requires a parent object which implements a 'getSettings' function
+      that returns a Settings object.
+    Call 'debug', 'info', 'warning', 'error' or 'critical' depending
+      on severity.
+    """
+
+    def debug(self, message, exc_info=True):
+        self.linkScopeLogger.debug(message, exc_info=exc_info)
+        return message
+
+    def info(self, message, popUp=False, exc_info=False):
+        self.linkScopeLogger.info(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.information(msgBox,
+                                              self.mainWindow.tr("Info"),
+                                              self.mainWindow.tr(message))
+        return message
+
+    def warning(self, message, popUp=False, exc_info=False):
+        self.linkScopeLogger.warning(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.warning(msgBox,
+                                          self.mainWindow.tr("Warning"),
+                                          self.mainWindow.tr(message))
+        return message
+
+    def error(self, message, popUp=True, exc_info=False):
+        self.linkScopeLogger.error(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.critical(msgBox,
+                                           self.mainWindow.tr("Error"),
+                                           self.mainWindow.tr(message))
+        return message
+
+    def critical(self, message, popUp=True, exc_info=True):
+        self.linkScopeLogger.critical(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.critical(msgBox,
+                                           self.mainWindow.tr("Critical"),
+                                           self.mainWindow.tr(message))
+        return message
+
+    # Set the severity level
+    def setSeverityLevel(self, level):
+        currentLogLevel = self.linkScopeLogger.level
+        try:
+            level = int(level)
+            self.linkScopeLogger.setLevel(level)
+            self.mainWindow.SETTINGS.setValue("Logging/Severity", self.linkScopeLogger.level)
+        except ValueError:
+            self.warning("Invalid Severity Level specified.")
+            self.linkScopeLogger.setLevel(currentLogLevel)
+            self.mainWindow.SETTINGS.setValue("Logging/Severity", currentLogLevel)
+
+    def getSeverityLevel(self):
+        return self.linkScopeLogger.level
+
+    def changeLogfile(self, newLogFile):
+        self.mainWindow.SETTINGS.setValue("Logging/Logfile", newLogFile)
+        self.logFileHandler = logging.FileHandler(
+            self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
+        for handler in self.linkScopeLogger.handlers[:]:
+            if isinstance(handler, logging.FileHandler):
+                self.linkScopeLogger.removeHandler(handler)
+        self.linkScopeLogger.addHandler(self.logFileHandler)
+
+    def __init__(self, parentObject):
+        self.mainWindow = parentObject
+        self.logQueue = Queue()
+        self.logFileHandler = logging.FileHandler(
+            self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
+        self.logQueueHandler = handlers.QueueHandler(self.logQueue)
+        self.logFormatter = logging.Formatter(
+            '{' + self.mainWindow.SETTINGS.value("Project/Name", "Untitled") + '} [%(asctime)s] - %(levelname)s: %('
+                                                                               'message)s')
+        self.logFormatterQueue = logging.Formatter('[%(asctime)s] - %(levelname)s: %(message)s')
+        self.logFileHandler.setFormatter(self.logFormatter)
+        self.logQueueHandler.setFormatter(self.logFormatterQueue)
+
+        # Do not handle root logger messages
+        rootLogger = logging.getLogger()
+        for handler in rootLogger.handlers:
+            if isinstance(handler, logging.Handler):
+                rootLogger.removeHandler(handler)
+
+        self.linkScopeLogger = logging.getLogger('LinkScope_Logger')
+        self.linkScopeLogger.addHandler(self.logFileHandler)
+        self.linkScopeLogger.addHandler(self.logQueueHandler)
+
+        self.setSeverityLevel(self.mainWindow.SETTINGS.value("Logging/Severity", logging.INFO))

Core/PathHelper.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+"""
+This file contains helper functions to validate paths on filesystems.
+Reference: https://stackoverflow.com/a/34102855
+
+Just call 'is_path_exists_or_creatable_portable' with the path (as a string) to validate it.
+"""
+
+import os
+import sys
+import tempfile
+import errno
+
+# Sadly, Python fails to provide the following magic number for us.
+ERROR_INVALID_NAME = 123
+'''
+Windows-specific error code indicating an invalid pathname.
+
+See Also
+----------
+https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
+    Official listing of all such codes.
+'''
+
+
+def is_pathname_valid(pathname: str) -> bool:
+    """
+    `True` if the passed pathname is a valid pathname for the current OS;
+    `False` otherwise.
+    """
+    # If this pathname is either not a string or is but is empty, this pathname
+    # is invalid.
+    try:
+        if not isinstance(pathname, str) or not pathname:
+            return False
+
+        # Strip this pathname's Windows-specific drive specifier (e.g., `C:\`)
+        # if any. Since Windows prohibits path components from containing `:`
+        # characters, failing to strip this `:`-suffixed prefix would
+        # erroneously invalidate all valid absolute Windows pathnames.
+        _, pathname = os.path.splitdrive(pathname)
+
+        # Directory guaranteed to exist. If the current OS is Windows, this is
+        # the drive to which Windows was installed (e.g., the "%HOMEDRIVE%"
+        # environment variable); else, the typical root directory.
+        root_dirname = os.environ.get('HOMEDRIVE', 'C:') \
+            if sys.platform == 'win32' else os.path.sep
+        assert os.path.isdir(root_dirname)  # ...Murphy and her ironclad Law  # nosec
+
+        # Append a path separator to this directory if needed.
+        root_dirname = root_dirname.rstrip(os.path.sep) + os.path.sep
+
+        # Test whether each path component split from this pathname is valid or
+        # not, ignoring non-existent and non-readable path components.
+        for pathname_part in pathname.split(os.path.sep):
+            try:
+                os.lstat(root_dirname + pathname_part)
+            # If an OS-specific exception is raised, its error code
+            # indicates whether this pathname is valid or not. Unless this
+            # is the case, this exception implies an ignorable kernel or
+            # filesystem complaint (e.g., path not found or inaccessible).
+            #
+            # Only the following exceptions indicate invalid pathnames:
+            #
+            # * Instances of the Windows-specific "WindowsError" class
+            #   defining the "winerror" attribute whose value is
+            #   "ERROR_INVALID_NAME". Under Windows, "winerror" is more
+            #   fine-grained and hence useful than the generic "errno"
+            #   attribute. When a too-long pathname is passed, for example,
+            #   "errno" is "ENOENT" (i.e., no such file or directory) rather
+            #   than "ENAMETOOLONG" (i.e., file name too long).
+            # * Instances of the cross-platform "OSError" class defining the
+            #   generic "errno" attribute whose value is either:
+            #   * Under most POSIX-compatible OSes, "ENAMETOOLONG".
+            #   * Under some edge-case OSes (e.g., SunOS, *BSD), "ERANGE".
+            except OSError as exc:
+                if hasattr(exc, 'winerror'):
+                    if exc.winerror == ERROR_INVALID_NAME:
+                        return False
+                elif exc.errno in {errno.ENAMETOOLONG, errno.ERANGE}:
+                    return False
+    # If a "TypeError" exception was raised, it almost certainly has the
+    # error message "embedded NUL character" indicating an invalid pathname.
+    except TypeError:
+        return False
+    # If no exception was raised, all path components and hence this
+    # pathname itself are valid. (Praise be to the curmudgeonly python.)
+    else:
+        return True
+    # If any other exception was raised, this is an unrelated fatal issue
+    # (e.g., a bug). Permit this exception to unwind the call stack.
+    #
+    # Did we mention this should be shipped with Python already?
+
+
+def is_path_sibling_creatable(pathname: str) -> bool:
+    """
+    `True` if the current user has sufficient permissions to create **siblings**
+    (i.e., arbitrary files in the parent directory) of the passed pathname;
+    `False` otherwise.
+    """
+    # Parent directory of the passed path. If empty, we substitute the current
+    # working directory (CWD) instead.
+    dirname = os.path.dirname(pathname) or os.getcwd()
+
+    try:
+        # For safety, explicitly close and hence delete this temporary file
+        # immediately after creating it in the passed path's parent directory.
+        with tempfile.TemporaryFile(dir=dirname):
+            pass
+        return True
+    # While the exact type of exception raised by the above function depends on
+    # the current version of the Python interpreter, all such types subclass the
+    # following exception superclass.
+    except EnvironmentError:
+        return False
+
+
+def is_path_exists_or_creatable_portable(pathname: str) -> bool:
+    """
+    `True` if the passed pathname is a valid pathname on the current OS _and_
+    either currently exists or is hypothetically creatable in a cross-platform
+    manner optimized for POSIX-unfriendly filesystems; `False` otherwise.
+
+    This function is guaranteed to _never_ raise exceptions.
+    """
+    try:
+        # To prevent "os" module calls from raising undesirable exceptions on
+        # invalid pathnames, is_pathname_valid() is explicitly called first.
+        return is_pathname_valid(pathname) and (
+                os.path.exists(pathname) or is_path_sibling_creatable(pathname))
+    # Report failure on non-fatal filesystem complaints (e.g., connection
+    # timeouts, permissions issues) implying this path to be inaccessible. All
+    # other exceptions are unrelated fatal issues and should not be caught here.
+    except OSError:
+        return False

Core/ReportGeneration.py

@@ -0,0 +1,478 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+from pathlib import Path
+from reportlab.lib import colors
+from reportlab.platypus.doctemplate import PageTemplate, BaseDocTemplate
+from reportlab.graphics.charts.piecharts import Pie
+from reportlab.platypus.frames import Frame
+from reportlab.platypus.tableofcontents import TableOfContents
+from reportlab.lib.units import cm
+
+from reportlab.pdfgen import canvas
+from reportlab.platypus import Paragraph, PageBreak, Image, Spacer, Table, ParagraphAndImage
+from reportlab.lib.enums import TA_LEFT, TA_CENTER, TA_JUSTIFY
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.pagesizes import LETTER, inch
+from reportlab.graphics.shapes import Line, Drawing
+
+from Core.GlobalVariables import avoid_parsing_fields
+
+
+class MyDocTemplate(BaseDocTemplate):
+    def __init__(self, filename, **kw):
+        self.allowSplitting = 0
+        BaseDocTemplate.__init__(self, filename, **kw)
+        template = PageTemplate('normal', [Frame(3.1 * cm, 2.5 * cm, 15 * cm, 25 * cm, id='F1')])
+        self.addPageTemplates(template)
+
+    def afterFlowable(self, flowable):
+        # Registers TOC entries.
+        if flowable.__class__.__name__ == 'Paragraph':
+            text = flowable.getPlainText()
+            style = flowable.style.name
+            if style == 'Heading1':
+                self.notify('TOCEntry', (0, text, self.page))
+            elif style == 'Heading2':
+                self.notify('TOCEntry', (1, text, self.page))
+
+
+class ReportBuilder(canvas.Canvas):
+
+    def __init__(self, *args, **kwargs):
+        canvas.Canvas.__init__(self, *args, **kwargs)
+        self.pages = []
+        self.width, self.height = LETTER
+
+    def showPage(self):
+        self.pages.append(dict(self.__dict__))
+        self._startPage()
+
+    def save(self):
+        pageCount = len(self.pages)
+        for page in self.pages:
+            self.__dict__.update(page)
+            if self._pageNumber > 1:
+                self.drawHeaderAndFooter(pageCount)
+            canvas.Canvas.showPage(self)
+        canvas.Canvas.save(self)
+
+    def drawHeaderAndFooter(self, pageCount):
+        pageCountString = f"Page {self._pageNumber} of {pageCount}"
+        self.saveState()
+        self.setStrokeColorRGB(0, 0, 0)
+        self.setLineWidth(0.5)
+        self.line(66, 78, LETTER[0] - 66, 78)
+        self.setFont('Times-Roman', 10)
+        self.drawString(LETTER[0] - 128, 65, pageCountString)
+        self.restoreState()
+
+
+class PDFReport:
+
+    def titlePage(self, authors, title, subtitle):  # arguments to be added: authors, title, subtitle
+        headline_style = self.styleSheet["Heading5"]
+        headline_style.alignment = TA_CENTER
+        headline_style.fontSize = 36
+        headline_style.leading = 50
+
+        subtitle_style = self.styleSheet["BodyText"]
+        subtitle_style.fontSize = 20
+        subtitle_style.leading = 30
+        subtitle_style.alignment = TA_CENTER
+
+        report_title = Paragraph('Investigation Report', headline_style)
+        self.elements.append(report_title)
+        spacer = Spacer(5, 5)
+        self.elements.append(spacer)
+
+        report_name = Paragraph(title, headline_style)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+        self.elements.append(report_name)
+
+        report_subtitle = Paragraph(subtitle, subtitle_style)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+        self.elements.append(report_subtitle)
+
+        spacer = Spacer(100, 425)
+        self.elements.append(spacer)
+
+        date = datetime.now().strftime('%Y-%m-%d %H:%m:%S')
+        paragraphStyleOptions = ParagraphStyle('Report', fontSize=9, leading=20, justifyBreaks=1, alignment=TA_LEFT,
+                                               justifyLastLine=1)
+        text = f"""Investigation Report<br/>
+        Authors: {authors}<br/>
+        Date: {date}<br/>
+        <br/>
+        """
+        paragraphReportSummary = Paragraph(text, paragraphStyleOptions)
+        self.elements.append(paragraphReportSummary)
+        self.elements.append(PageBreak())
+
+    def summaryPage(self, userSummary, summaryCanvasImage):  # arguments: user summary input
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        paragraphStyleOptions = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_JUSTIFY,
+                                               justifyLastLine=0)
+        text = userSummary
+        paragraphReportSummary = Paragraph(text, paragraphStyleOptions)
+
+        # path to image needed
+        img = Image(summaryCanvasImage)
+        img.preserveAspectRatio = True
+        img._restrictSize(6.5 * inch, 5.5 * inch)
+        # img.drawHeight = 5.5 * inch
+        # img.drawWidth = 6.5 * inch
+        img.hAlign = 'CENTER'
+
+        self.elements.append(paragraphReportSummary)
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+
+        self.elements.append(img)
+        self.elements.append(PageBreak())
+
+    def entityPage(self, userNotes, entityImagePath, appendixDicts, outgoingLinks, incomingLinks, entity,
+                   outgoingNames, incomingNames, title, appendixNumber):
+        # dict with image and notes for appendix
+
+        COLORBLACK = colors.HexColor(0x241F20)
+
+        entity_notes_header = ''
+        entity_notes = ''
+
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        entity_style = self.styleSheet["Title"]
+        entity_style.fontSize = 24
+        entity_style.leading = 20
+        entity_style.alignment = TA_CENTER
+        entity_style.fontName = 'Times-Bold'
+
+        psSubHeaderText = ParagraphStyle('Heading2', fontSize=12, alignment=TA_CENTER, borderWidth=1,
+                                         fontName='Times-Bold')
+        notesParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=0, alignment=TA_JUSTIFY, justifyLastLine=0)
+
+        entityTitle = Paragraph(title, entity_style)
+
+        notes = f"""
+        <font name="Times-Bold" size="12"> User's Notes:</font><br/> {userNotes} 
+
+        """
+
+        text = Paragraph(notes, notesParagraph)
+        if isinstance(entityImagePath, Drawing):
+            tbl = ReportImageAndParagraph(text, entityImagePath, side='left', xpad=10, ypad=0)
+        elif entityImagePath.endswith('.png') or entityImagePath.endswith('.jpg'):
+            entityImage = Image(entityImagePath)
+            entityImage.preserveAspectRatio = True
+            entityImage.drawHeight = 2 * inch
+            entityImage.drawWidth = 2 * inch
+            tbl = ReportImageAndParagraph(text, entityImage, side='left', xpad=10, ypad=0)
+        else:
+            raise ValueError('Invalid Image Type')
+
+        links_subHeader = Paragraph("Entity Links", psSubHeaderText)
+
+        entity_summary_subHeader = Paragraph("Entity Summary", psSubHeaderText)
+
+        links_table_style = [('GRID', (0, 0), (-1, -1), 1, COLORBLACK), ('SPAN', (3, 0), (0, 0)),
+                             ('LINEABOVE', (0, 2), (-1, 2), 1, colors.blue),
+                             ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
+                             ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
+
+        # child UID
+        if outgoingLinks:
+            outgoing_data = [
+                ['Outgoing Links'],
+                ['Resolution Name', 'Child Entity', 'Date Created', 'Notes']]
+            for index, link in enumerate(outgoingLinks):
+                resolutionText = link['Resolution']
+                childNodeText = outgoingNames[index]
+                linkNotesText = link['Notes']
+                dateCreatedText = link['Date Created']
+                linkName = "".join([resolutionText[counter:counter + 24] + "\n"
+                                    for counter in range(0, len(resolutionText), 24)])
+                childNode = "".join([childNodeText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(childNodeText), 24)])
+                linkNotes = "".join([linkNotesText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(linkNotesText), 24)])
+                dateCreated = "".join([dateCreatedText[counter:counter + 24] + "\n"
+                                       for counter in range(0, len(dateCreatedText), 24)])
+                outgoing_data.append([linkName, childNode, dateCreated, Paragraph(linkNotes)])
+            outgoing_table = Table(data=outgoing_data, style=links_table_style, hAlign="CENTER",
+                                   colWidths=[140, 140, 140, 140])
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+        else:
+            outgoing_data = [
+                ['No Outgoing Links']]
+            outgoing_table = Table(data=outgoing_data, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+        # parent UID
+        if incomingLinks:
+            incoming_data = [
+                ['Incoming Links'],
+                ['Resolution Name', 'Parent Entity', 'Date Created', 'Notes']]
+            for index, link in enumerate(incomingLinks):
+                resolutionText = link['Resolution']
+                parentNodeText = incomingNames[index]
+                linkNotesText = link['Notes']
+                dateCreatedText = link['Date Created']
+                linkName = "".join([resolutionText[counter:counter + 24] + "\n"
+                                    for counter in range(0, len(resolutionText), 24)])
+                parentNode = "".join([parentNodeText[counter:counter + 24] + "\n"
+                                      for counter in range(0, len(parentNodeText), 24)])
+                linkNotes = "".join([linkNotesText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(linkNotesText), 24)])
+                dateCreated = "".join([dateCreatedText[counter:counter + 24] + "\n"
+                                       for counter in range(0, len(dateCreatedText), 24)])
+                incoming_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
+            incoming_table = Table(data=incoming_data, style=links_table_style, hAlign="CENTER",
+                                   colWidths=[140, 140, 140, 140])
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+        else:
+            incoming_data = [
+                ['No Incoming Links']]
+            incoming_table = Table(data=incoming_data, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+        tableParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_CENTER,
+                                        justifyLastLine=0)
+
+        ParagraphStyle('Heading2', fontSize=11, justifyBreaks=1, alignment=TA_CENTER,
+                       justifyLastLine=1, fontName='Times-Bold', rightIndent=45)
+        notesHeader = ParagraphStyle('Heading2', fontSize=11, justifyBreaks=1, alignment=TA_CENTER,
+                                     justifyLastLine=1, fontName='Times-Bold')
+        entity_data = [
+            ['Entity Fields'],
+            ['Attribute', 'Value']]
+        entity_table_style = [('GRID', (0, 0), (-1, -1), 1, COLORBLACK), ('SPAN', (1, 0), (0, 0)),
+                              ('LINEABOVE', (0, 2), (-1, 2), 1, colors.blue),
+                              ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
+                              ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
+        for key in list(entity):
+            if key not in avoid_parsing_fields and key != 'Notes':
+                valueText = entity[key]
+                value = "".join([valueText[counter:counter + 80] + "\n"
+                                 for counter in range(0, len(valueText), 80)])
+                entity_data.append([Paragraph(key, tableParagraph), Paragraph(value, tableParagraph)])
+            elif key == 'Notes':
+                entity_notes_header = Paragraph("Entity Notes ", notesHeader)
+                entity_notes = Paragraph(entity[key], notesParagraph)
+        entity_table = Table(data=entity_data, style=entity_table_style, hAlign="CENTER")
+
+        self.elements.append(entityTitle)
+        spacer = Spacer(20, 20)
+        self.elements.append(spacer)
+
+        self.elements.append(tbl)
+        # self.elements.append(img)
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+        self.elements.append(links_subHeader)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(incoming_table)
+        spacer = Spacer(25, 25)
+        self.elements.append(spacer)
+        self.elements.append(outgoing_table)
+
+        if not incomingLinks and not outgoingLinks:
+            # No links to draw link pie graph.
+            pass
+        else:
+            pieDraw = Drawing()
+            pie = Pie()
+            pie.x = 150
+            pie.y = 65
+            pie.data = [len(incomingLinks), len(outgoingLinks)]
+            pie.sideLabels = 1
+            pie.labels = ['Incoming: ' + str(len(incomingLinks)), 'Outgoing: ' + str(len(outgoingLinks))]
+            pie.slices.strokeWidth = 1
+            if len(incomingLinks) > len(outgoingLinks):
+                pie.slices[0].popout = 5
+            else:
+                pie.slices[1].popout = 5
+            pieDraw.add(pie)
+            self.elements.append(pieDraw)
+
+        self.elements.append(entity_summary_subHeader)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(entity_table)
+        self.elements.append(spacer)
+        self.elements.append(entity_notes_header)
+        self.elements.append(entity_notes)
+        self.elements.append(spacer)
+
+        spacer = Spacer(20, 20)
+        for appendixIndex, appendixDict in enumerate(appendixDicts):
+            appendix_header = Paragraph(f"Entity Appendix {appendixIndex}", notesHeader)
+            self.elements.append(appendix_header)
+            self.elements.append(spacer)
+
+            appendixImage = appendixDict['AppendixEntityImage']
+            appendixNotes = appendixDict['AppendixEntityNotes']
+            if appendixImage == '':
+                self.elements.append(Paragraph(appendixNotes, notesParagraph))
+            elif appendixNotes == '' and appendixImage != '':
+                img = Image(Path(appendixImage))
+                # img.preserveAspectRatio = True
+                img.drawHeight = 3 * inch
+                img.drawWidth = 3 * inch
+                self.elements.append(img)
+            elif appendixNotes != '' and appendixImage != '':
+                paragraph = appendixNotes
+                img = Image(Path(appendixImage))
+                img.drawHeight = 3 * inch
+                img.drawWidth = 3 * inch
+                self.elements.append(ReportImageAndParagraph(Paragraph(paragraph), img, side='left', xpad=10, ypad=0))
+            self.elements.append(spacer)
+
+        self.elements.append(PageBreak())
+
+    # Currently, does not really help very much and does not look good, so the graph page is not generated.
+    def graphPage(self, timeLineImage):
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_LEFT,
+                       justifyLastLine=1)
+
+        img = Image(timeLineImage)
+        img.drawHeight = 1.3 * inch
+        img.drawWidth = 6 * inch
+        img.hAlign = 'LEFT'
+
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+        self.elements.append(img)
+        self.elements.append(PageBreak())
+
+    def nextPagesHeader(self, isSecondPage, header):
+        if isSecondPage:
+            psHeaderText = ParagraphStyle('Heading1', fontSize=14, alignment=TA_LEFT, borderWidth=1)
+            h = header
+            paragraphReportHeader = Paragraph(h, psHeaderText)
+            self.elements.append(paragraphReportHeader)
+
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+            d = Drawing(500, 1)
+            line = Line(-15, 0, 430, 0)
+            line.strokeWidth = 2
+            d.add(line)
+            self.elements.append(d)
+
+            spacer = Spacer(10, 1)
+            self.elements.append(spacer)
+
+            d = Drawing(500, 1)
+            line = Line(-15, 0, 430, 0)
+            line.strokeWidth = 0.5
+            d.add(line)
+            self.elements.append(d)
+
+    def __init__(self, path: str, entityListData, outgoingLinks, incomingLinks, entity, summaryCanvasImage,
+                 timelineImage, entityPrimaryField, incomingNames, outgoingNames):
+        self.path = path
+        self.styleSheet = getSampleStyleSheet()
+        self.elements = []
+
+        self.titlePage(authors=entityListData[1].get('Authors'), title=entityListData[1].get('Title'),
+                       subtitle=entityListData[1].get('Subtitle'))
+
+        tocHeaderStyle = ParagraphStyle('Title', fontSize=16, alignment=TA_CENTER, borderWidth=1)
+        tocTitle = Paragraph('Table of Contents', tocHeaderStyle)
+        h1 = ParagraphStyle(name='Heading1',
+                            fontSize=14,
+                            leading=16)
+        h2 = ParagraphStyle(name='Heading2',
+                            fontSize=12,
+                            leading=14, )
+
+        toc = TableOfContents()
+        toc.levelStyles = [h1, h2]
+        self.elements.append(tocTitle)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(toc)
+        self.elements.append(PageBreak())
+
+        head = 'Summary Report'
+        self.nextPagesHeader(True, head)
+
+        self.summaryPage(userSummary=entityListData[2].get('SummaryNotes'), summaryCanvasImage=summaryCanvasImage)
+
+        imagePath = ''
+        for i in range(3, len(entityListData)):
+            if entityListData[i][0].get('EntityImage') != '':
+                imagePath = entityListData[i][0].get('EntityImage')
+
+            head = f'Entity Report: {entityPrimaryField[i - 3]}'
+            self.nextPagesHeader(True, head)
+            self.entityPage(title=entityPrimaryField[i - 3],
+                            userNotes=entityListData[i][0].get('EntityNotes'),
+                            entityImagePath=imagePath,
+                            appendixDicts=entityListData[i][1],
+                            outgoingLinks=outgoingLinks[i - 3],
+                            incomingLinks=incomingLinks[i - 3],
+                            entity=entity[i - 3],
+                            incomingNames=incomingNames[i - 3],
+                            outgoingNames=outgoingNames[i - 3],
+                            appendixNumber=i - 3)
+
+        # Graph report stuff disabled, at least for now.
+        # head = 'Graph Report'
+        # self.nextPagesHeader(True, head)
+        # self.graphPage(timelineImage)
+
+        # Build
+        self.doc = MyDocTemplate(path)
+        self.doc.multiBuild(self.elements, canvasmaker=ReportBuilder)
+
+
+# Subclass ParagraphAndImage to adjust the alignment of the text with the image in the element.
+class ReportImageAndParagraph(ParagraphAndImage):
+
+    def __init__(self, P, I, xpad=3, ypad=3, side='right'):
+        super().__init__(P, I, xpad, ypad, side)
+        self.hI = ''
+        self.wI = ''
+        self._offsets = ''
+
+    def wrap(self, availWidth, availHeight):
+        wI, hI = self.I.wrap(availWidth, availHeight)
+        self.hI = hI
+        self.wI = wI
+        # work out widths array for breaking
+        self.width = availWidth
+        P = self.P
+        style = P.style
+        xpad = self.xpad
+        ypad = self.ypad
+        leading = style.leading
+        leftIndent = style.leftIndent
+        later_widths = availWidth - leftIndent - style.rightIndent
+        intermediate_widths = later_widths - xpad - wI
+        first_line_width = intermediate_widths - style.firstLineIndent
+        P.width = 0
+        nIW = int((hI + ypad) / (leading * 1.0))
+        P.blPara = P.breakLines([first_line_width] + nIW * [intermediate_widths] + [later_widths])
+        if self._side == 'left':
+            self._offsets = [wI + xpad] * (1 + nIW) + [0]
+        # Make the paragraph line up with the top part of the image if its height is less than
+        #   the height of the image.
+        P.height = max(hI, len(P.blPara.lines) * leading)
+        self.height = max(hI, P.height)
+        return self.width, self.height

Core/ResolutionManager.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+
+import contextlib
+import importlib.util
+import sys
+from os import listdir
+from pathlib import Path
+from uuid import uuid4
+from typing import Union
+
+
+class ResolutionManager:
+
+    # Load all resources needed.
+    def __init__(self, mainWindow, messageHandler):
+        self.messageHandler = messageHandler
+        self.mainWindow = mainWindow
+        self.resolutions = {}
+        # Macro dict item contents: tuple of (resolution, parameter values)
+        self.macros = {}
+
+    def loadResolutionsFromDir(self, directory: Path) -> None:
+        exceptionsCount = 0
+        for resolution in listdir(directory):
+            resolution = str(resolution)
+            try:
+                if resolution.endswith('.py'):
+                    resolutionName = resolution[:-3]
+                    spec = importlib.util.spec_from_file_location(
+                        resolutionName,
+                        directory / resolution)
+                    module = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(module)
+                    resClass = getattr(module, resolutionName)
+                    resClassInst = resClass()
+                    resNameString = resClassInst.name.replace('_', ' ')
+                    resolutionDesc = resClassInst.description
+                    originTypes = resClassInst.originTypes
+                    resultTypes = resClassInst.resultTypes
+                    resolutionParameters = resClassInst.parameters
+                    with contextlib.suppress(AttributeError):
+                        resolutionCategory = resClassInst.category
+                        if not isinstance(resolutionCategory, str):
+                            resolutionCategory = "Uncategorized"
+                    if self.resolutions.get(resolutionCategory) is None:
+                        self.resolutions[resolutionCategory] = {}
+                    self.resolutions[resolutionCategory][resNameString] = {'name': resNameString,
+                                                                           'description': resolutionDesc,
+                                                                           'originTypes': originTypes,
+                                                                           'resultTypes': resultTypes,
+                                                                           'parameters': resolutionParameters,
+                                                                           'category': resolutionCategory,
+                                                                           'resolution': resClass
+                                                                           }
+                    self.messageHandler.info(f"Loaded Resolution: {resNameString}")
+            except Exception as e:
+                self.messageHandler.error(f"Cannot load resolutions from {str(directory)}" + "\n Info: " + repr(e))
+                exceptionsCount += 1
+                if exceptionsCount > 3:
+                    # Will not occur when loading modules with 3 or fewer resolutions, but that should be fine.
+                    self.messageHandler.critical("Failed loading too many resolutions to proceed.")
+                    sys.exit(5)
+
+    def getResolutionParameters(self, resolutionCategory, resolutionNameString):
+        resolutionsList = self.resolutions.get(resolutionCategory)
+        if resolutionsList is not None and resolutionNameString in resolutionsList:
+            return self.resolutions[resolutionCategory][resolutionNameString]['parameters']
+        return None
+
+    def getResolutionOriginTypes(self, resolutionCategoryNameString: str) -> Union[list, None]:
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                originTypes = self.resolutions[resolutionCategory][resolutionName]['originTypes']
+                if '*' in originTypes:
+                    originTypes = self.mainWindow.RESOURCEHANDLER.getAllEntities()
+                return originTypes
+        return None
+
+    def getResolutionDescription(self, resolutionCategoryNameString: str) -> Union[str, None]:
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                return self.resolutions[resolutionCategory][resolutionName].get('description', '')
+        return None
+
+    def loadResolutionsFromServer(self, serverRes) -> None:
+        for category in serverRes:
+            if category not in self.resolutions:
+                self.resolutions[category] = {}
+            for serverResolution in serverRes[category]:
+                self.resolutions[category][serverResolution] = serverRes[category][serverResolution]
+
+    def removeServerResolutions(self) -> None:
+        for category in dict(self.resolutions):
+            for resolution in dict(self.resolutions[category]):
+                # If resolution class does not exist locally, then assume it exists on the server.
+                if self.resolutions[category][resolution]['resolution'] == '':
+                    self.resolutions[category].pop(resolution)
+        for category in dict(self.resolutions):
+            if len(self.resolutions[category]) == 0:
+                self.resolutions.pop(category)
+
+    def getResolutionCategories(self) -> list:
+        return list(self.resolutions)
+
+    def getResolutionsForEntityTypesByCategory(self, eTypes) -> dict:
+        """
+        Gets a set of entity types, and returns a dictionary with all the resolutions that can take all
+        included types as input.
+        If '*' is part of eTypes, get all Resolutions of all categories.
+        """
+        result = {}
+        for category in self.resolutions:
+            result[category] = []
+            for resolution in self.resolutions[category]:
+                originTypes = self.resolutions[category][resolution]['originTypes']
+                if eTypes.issubset(originTypes) or '*' in originTypes:
+                    result[category].append(resolution)
+        return result
+
+    def getResolutionsInCategory(self, category) -> list:
+        return list(self.resolutions[category]) if category in self.resolutions else []
+
+    def getAllResolutions(self) -> list:
+        categories = self.getResolutionCategories()
+        result = []
+        for category in categories:
+            result += self.getResolutionsInCategory(category)
+        return result
+
+    def executeResolution(self, resolutionCategoryNameString: str, resolutionEntitiesInput: list, parameters: dict,
+                          resolutionUID: str):
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                if self.resolutions[resolutionCategory][resolutionName].get('resolution') == '':
+                    # If resolution class does not exist locally, then assume it exists on the server.
+                    self.mainWindow.executeRemoteResolution(resolutionCategoryNameString, resolutionEntitiesInput,
+                                                            parameters, resolutionUID)
+                    # Returning a bool, so we know that the resolution is running on the server.
+                    return True
+                resolutionClass = self.resolutions[resolutionCategory][resolutionName]['resolution']()
+                return resolutionClass.resolution(resolutionEntitiesInput, parameters)
+        return None
+
+    def createMacro(self, resolutionList: list) -> str:
+        macroUID = str(uuid4())
+        self.macros[macroUID] = resolutionList
+
+        return macroUID
+
+    def renameMacro(self, oldName: str, newName: str) -> bool:
+        if oldName != newName:
+            with self.mainWindow.macrosLock:
+                if newName in self.macros:
+                    self.mainWindow.MESSAGEHANDLER.warning('The specified name already exists. '
+                                                           'Macro names must be unique.',
+                                                           popUp=True)
+                    return False
+                if oldName not in self.macros:
+                    self.mainWindow.MESSAGEHANDLER.error('Attempting to rename a nonexistent macro.', popUp=True)
+                    return False
+                oldMacro = self.macros.pop(oldName)
+                self.macros[newName] = oldMacro
+        return True
+
+    def deleteMacro(self, macroUID: str) -> bool:
+        # We don't have to worry about running macros, because the details of the macro are saved in memory.
+        # We do however want to get the thread lock because of potential race conditions.
+        try:
+            with self.mainWindow.macrosLock:
+                self.macros.pop(macroUID)
+            return True
+        except KeyError:
+            return False
+
+    def loadMacros(self) -> None:
+        # Load AFTER we load resolutions.
+        self.macros = self.mainWindow.SETTINGS.value("Program/Macros", {})
+
+    def save(self) -> None:
+        self.mainWindow.SETTINGS.setGlobalValue("Program/Macros", self.macros)

Core/Resolutions/Core/ASNToCIDR.py

@@ -0,0 +1,58 @@
+class ASNToCIDR:
+    # A string that is treated as the name of this resolution.
+    name = "Get CIDR from ASN"
+    category = "Network Infrastructure"
+
+    # A string that describes this resolution.
+    description = "Get network information for an Autonomous System Number."
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'Autonomous System'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Network'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois.net import Net
+        from ipwhois.asn import ASNOrigin
+
+        returnResult = []
+
+        for entity in entityJsonList:
+            if entity['Entity Type'] == 'Autonomous System':
+                try:
+                    ipWithPrefix = entity['ASN Cidr']
+                    uid = entity['uid']
+                    split_string = ipWithPrefix.split("/", 1)
+                    ipWithOutPrefix = split_string[0]
+                    ASN = entity["AS Number"]
+                except Exception:
+                    continue
+                try:
+                    net = Net(ipWithOutPrefix)
+                except Exception:
+                    net = Net('1.1.1.1')
+                obj = ASNOrigin(net)
+                results = obj.lookup(asn=ASN)
+
+                for network in results['nets']:
+                    cidrWithPrefix = network['cidr']
+                    split_string = cidrWithPrefix.split("/", 1)
+                    cidrWithOutPrefix = split_string[0]
+                    prefix = split_string[1]
+                    index_of_child = len(returnResult)
+                    returnResult.extend(([{'IP Address': cidrWithOutPrefix, 'Range': prefix, 'Entity Type': 'Network'},
+                                          {uid: {'Resolution': 'ASN to CIDR', 'Notes': ''}}],
+                                         [{'Phrase': network['description'], 'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'CIDR Description', 'Notes': ''}}],
+                                         [{'Organization Name': network['source'], 'Entity Type': 'Organization'},
+                                          {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}],
+                                         [{'Company Name': network['maintainer'], 'Entity Type': 'Company'},
+                                          {index_of_child: {'Resolution': 'Company Name', 'Notes': ''}}]))
+
+        return returnResult

Core/Resolutions/Core/AffiliateCodesExtractor.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+
+"""
+This resolution can, in rare instances, be a bit unreliable - if it finishes immediately, it's possible that not all
+links were considered, as the page's javascript may not have completely finished loading.
+
+Also, since as far as I can tell one cannot mute the audio of playwright, if a site that autoplays video / audio
+is explored, it is possible that some audio plays while the page is loading and its contents are processed.
+"""
+
+
+class AffiliateCodesExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Affiliate Codes"
+
+    category = "Website Tracking"
+
+    # A string that describes this resolution.
+    description = "Returns Facebook and Amazon affiliate codes found in websites."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'},
+                  'Visit External Links': {'description': 'Affiliate codes can be hidden in shortened links, and they '
+                                                          'might not be visible unless one visits the site directly. '
+                                                          'Visiting external links should result in more affiliate '
+                                                          'tags being extracted. Of course, browsing to random pages '
+                                                          'is not always safe.\nVisit external links?',
+                                           'type': 'SingleChoice',
+                                           'value': {'Yes', 'No'}}}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import urllib
+        import tldextract
+        import re
+
+        returnResults = []
+        visitExternal = parameters['Visit External Links'] == 'Yes'
+
+        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        # Sites like youtube replace external links with a redirect link originating
+        #   from the site itself. This sort of gets around that.
+        redirectRegex = re.compile(r'q=[^\s][^&^#]*', re.IGNORECASE)
+
+        amazonRegex = re.compile(r'tag=[^\s][^&]*', re.IGNORECASE)
+        fbRegex = re.compile(r'client_id=\d{4,25}', re.IGNORECASE)
+        webullRegex = re.compile(r'inviteCode=[^\s][^&]*', re.IGNORECASE)
+        smartpassiveincomeRegex = re.compile(r'affcode=[^\s][^&]*', re.IGNORECASE)
+        skillshareRegex = re.compile(r'utm_campaign=[^\s][^&]*', re.IGNORECASE)
+        freetradeRegex = re.compile(r'https://freetrade.app.link/[^\s][^?]*', re.IGNORECASE)
+        freetradeRegex2 = re.compile(r'https://magic.freetrade.io/join/[^\s][^?^#]*', re.IGNORECASE)
+
+        exploredDepth = set()
+        exploredForeign = set()
+        alreadyParsed = set()
+
+        def GetAffiliateCodes(currentUID: str, site: str):
+            requestUrl = str(urllib.parse.unquote(site))
+            if requestUrl not in alreadyParsed:
+                alreadyParsed.add(requestUrl)
+                for affiliateCode in amazonRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Amazon Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in fbRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Facebook Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in webullRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'WeBull Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in smartpassiveincomeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'SmartPassiveIncome Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in skillshareRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'SkillShare Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in freetradeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'FreeTrade Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in freetradeRegex2.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'FreeTrade Affiliate Code',
+                                                        'Notes': ''}}])
+
+        def extractCodes(currentUID: str, site: str, depth: int):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            linksInLinkHref = soupContents.find_all('link')
+            for tag in linksInLinkHref:
+                newLink = tag.get('href', None)
+                if newLink is not None and newLink.startswith('http'):
+                    newLink = newLink.split('#')[0]
+                    newDepth = depth - 1
+                    if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                        exploredDepth.add(newLink)
+                        extractCodes(currentUID, newLink, newDepth)
+
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None and newLink.startswith('http'):
+                    newLink = newLink.split('#')[0]
+                    newDepth = depth - 1
+                    if domain in newLink:
+                        redirLinks = redirectRegex.findall(newLink)
+                        if 'redirect' in newLink and len(redirLinks) > 0:
+                            newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
+                            if newLink not in exploredForeign:
+                                exploredForeign.add(newLink)
+                                if visitExternal:
+                                    for _ in range(3):
+                                        try:
+                                            page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                            GetAffiliateCodes(currentUID, page.url)
+                                            break
+                                        except TimeoutError:
+                                            pass
+                                        except Error:
+                                            break
+                            else:
+                                GetAffiliateCodes(currentUID, newLink)
+                        else:
+                            if newLink not in exploredDepth and newDepth > 0:
+                                exploredDepth.add(newLink)
+                                extractCodes(currentUID, newLink, newDepth)
+                    elif newLink not in exploredForeign:
+                        exploredForeign.add(newLink)
+                        if visitExternal:
+                            for _ in range(3):
+                                try:
+                                    page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                    GetAffiliateCodes(currentUID, page.url)
+                                    break
+                                except TimeoutError:
+                                    pass
+                                except Error:
+                                    break
+                        else:
+                            GetAffiliateCodes(currentUID, newLink)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                domain = tldextract.extract(url).fqdn
+                extractCodes(uid, url, maxDepth)
+            browser.close()
+        return returnResults

Core/Resolutions/Core/CertificateInfo.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+
+
+class CertificateInfo:
+    name = "Analyze Website Certificate"
+    category = "Network Infrastructure"
+    description = "Get certificate information from the certificate of a certain website."
+    originTypes = {'Website'}
+    resultTypes = {'Phrase', 'Website', 'Domain', 'Country', 'Organization', 'Address', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        import ssl
+        import socket
+
+        returnResults = []
+        sslContext = ssl.create_default_context()
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            websiteURL = entity['URL']
+            if websiteURL.startswith('https'):
+                addressWithoutHttps = websiteURL[8:].split('/')[0]
+            elif websiteURL.startswith('http'):
+                # Just in case an entity was created for the http version of a site,
+                #   but there is actually a https version of the site.
+                addressWithoutHttps = websiteURL[7:].split('/')[0]
+            else:
+                continue
+
+            with sslContext.wrap_socket(socket.socket(), server_hostname=addressWithoutHttps) as s:
+                try:
+                    s.connect((addressWithoutHttps, 443))
+                    websiteCertificate = s.getpeercert()
+                except socket.error:
+                    # If there's an error connecting, move on.
+                    continue
+
+            # Subject Details
+            streetAddr = None
+            locality = None
+            postalCode = None
+            subjectCountry = None
+            subjectSerial = None
+            subjectName = None
+            tentativeSubjectResults = []
+            for subjectAttributeOuter in websiteCertificate['subject']:
+                for subjectAttributeInner in subjectAttributeOuter:
+                    subjectAttributeInnerKey = subjectAttributeInner[0]
+                    subjectAttributeInnerValue = subjectAttributeInner[1]
+                    if subjectAttributeInnerKey == 'commonName':
+                        returnResults.append([{'Domain Name': subjectAttributeInnerValue,
+                                               'Entity Type': 'Domain'},
+                                              {uid: {'Resolution': 'Certificate Subject Common Name',
+                                                     'Notes': ''}}])
+                    elif subjectAttributeInnerKey == 'countryName':
+                        subjectCountry = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'localityName':
+                        locality = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'organizationName':
+                        subjectName = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'postalCode':
+                        postalCode = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'serialNumber':
+                        subjectSerial = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'streetAddress':
+                        streetAddr = subjectAttributeInnerValue
+            subjectIndex = None
+            if subjectName is not None:
+                subjectIndex = len(returnResults)
+                subjectNameJSON = {'Organization Name': subjectName,
+                                   'Entity Type': 'Organization'}
+                if subjectSerial is not None:
+                    subjectNameJSON['Registration Number'] = subjectSerial
+                tentativeSubjectResults.append([subjectNameJSON,
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we only have the serial of the subject, use that as the subject entity representation.
+            elif subjectSerial is not None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Phrase': subjectSerial,
+                                                 'Entity Type': 'Phrase'},
+                                                {uid: {'Resolution': 'Certificate Subject Serial Number',
+                                                       'Notes': ''}}])
+
+            # If we have no company name and no serial, the certificate belongs to an unknown org.
+            if subjectIndex is None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Organization Name': 'Unknown Organization',
+                                                 'Entity Type': 'Organization'},
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we have an address, fill it in. If not, create separate entities for each element.
+            if streetAddr is not None:
+                streetAddressJSON = {'Street Address': streetAddr,
+                                     'Entity Type': 'Address'}
+                if subjectCountry is not None:
+                    streetAddressJSON['Country'] = subjectCountry
+                if postalCode is not None:
+                    streetAddressJSON['Postal Code'] = postalCode
+                if locality is not None:
+                    streetAddressJSON['Locality'] = locality
+                tentativeSubjectResults.append([streetAddressJSON,
+                                                {subjectIndex: {'Resolution': 'Certificate Subject Address',
+                                                                'Notes': ''}}])
+            else:
+                if subjectCountry is not None:
+                    tentativeSubjectResults.append([{'Country Name': subjectCountry,
+                                                     'Entity Type': 'Country'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Country',
+                                                                    'Notes': ''}}])
+                if postalCode is not None:
+                    tentativeSubjectResults.append([{'Phrase': postalCode,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Postal Code',
+                                                                    'Notes': ''}}])
+                if locality is not None:
+                    tentativeSubjectResults.append([{'Phrase': locality,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Locality',
+                                                                    'Notes': ''}}])
+            if len(tentativeSubjectResults) > 1:
+                # If we have more than just 'Unknown Organization' as a result for the subject, consider the output.
+                #   Otherwise, no point in including it.
+                returnResults += tentativeSubjectResults
+
+            # Validity dates for the certificate
+            returnResults.append([{'Date': websiteCertificate['notBefore'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Start Date',
+                                         'Notes': ''}}])
+            returnResults.append([{'Date': websiteCertificate['notAfter'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Expiry Date',
+                                         'Notes': ''}}])
+
+            # Domain names included in the certificate.
+            with contextlib.suppress(KeyError):
+                for altNameAttribute in websiteCertificate['subjectAltName']:
+                    returnResults.append([{'Domain Name': altNameAttribute[1],
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Certificate Subject Alternate Name',
+                                                 'Notes': ''}}])
+            # OCSP URLs. Often just one.
+            with contextlib.suppress(KeyError):
+                for ocsp in websiteCertificate['OCSP']:
+                    returnResults.append([{'URL': ocsp,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate OCSP URL',
+                                                 'Notes': ''}}])
+            # CA Issuer URL
+            with contextlib.suppress(KeyError):
+                for caIssuer in websiteCertificate['caIssuers']:
+                    returnResults.append([{'URL': caIssuer,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Issuer URL',
+                                                 'Notes': ''}}])
+            # CRL URLs
+            with contextlib.suppress(KeyError):
+                for crlDistributionPoint in websiteCertificate['crlDistributionPoints']:
+                    returnResults.append([{'URL': crlDistributionPoint,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Revocation List URL',
+                                                 'Notes': ''}}])
+            # Issuer information
+            orgName = None
+            orgCommonName = None
+            orgCountry = None
+            orgPostal = None
+            orgLocality = None
+            orgStateOrProvince = None
+            orgUnitName = None
+            for issuerAttributeOuter in websiteCertificate['issuer']:
+                for issuerAttributeInner in issuerAttributeOuter:
+                    issuerAttributeInnerKey = issuerAttributeInner[0]
+                    issuerAttributeInnerValue = issuerAttributeInner[1]
+                    if issuerAttributeInnerKey == 'commonName':
+                        orgCommonName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'countryName':
+                        orgCountry = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'localityName':
+                        orgLocality = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'orgUnitName':
+                        orgUnitName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'organizationName':
+                        orgName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'postalCode':
+                        orgPostal = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'stateOrProvinceName':
+                        orgStateOrProvince = issuerAttributeInnerValue
+            issuerIndex = None
+            if orgName is not None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': orgName,
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Organization',
+                                             'Notes': ''}}])
+            if orgCommonName is not None:
+                if issuerIndex is not None:
+                    parentEntity = issuerIndex
+                else:
+                    parentEntity = uid
+                    issuerIndex = len(returnResults)
+                returnResults.append([{'Phrase': orgCommonName,
+                                       'Entity Type': 'Phrase'},
+                                      {parentEntity: {'Resolution': 'Certificate Issuer Common Name',
+                                                      'Notes': ''}}])
+            if issuerIndex is None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': 'Unknown Issuer',
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Name',
+                                             'Notes': ''}}])
+            if orgCountry is not None:
+                returnResults.append([{'Country Name': orgCountry,
+                                       'Entity Type': 'Country'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Country',
+                                                     'Notes': ''}}])
+            if orgPostal is not None:
+                returnResults.append([{'Phrase': orgPostal,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Postal Code',
+                                                     'Notes': ''}}])
+            if orgLocality is not None:
+                returnResults.append([{'Phrase': orgLocality,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Locality',
+                                                     'Notes': ''}}])
+            if orgStateOrProvince is not None:
+                returnResults.append([{'Phrase': orgStateOrProvince,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer State / Province',
+                                                     'Notes': ''}}])
+            if orgUnitName is not None:
+                returnResults.append([{'Phrase': orgUnitName,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Organizational Unit Name',
+                                                     'Notes': ''}}])
+
+            # Certificate info - serial number and certificate version.
+            returnResults.append([{'Phrase': websiteCertificate['serialNumber'],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Serial Number',
+                                         'Notes': ''}}])
+
+            returnResults.append([{'Phrase': 'Certificate Version: ' + str(websiteCertificate['version']),
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Version',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ContainsPhrase.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+
+class ContainsPhrase:
+    # A string that is treated as the name of this resolution.
+    name = "Contains Phrase"
+
+    category = "String Operations"
+
+    # A string that describes this resolution.
+    description = "Checks if a Phrase exists in the Notes of a Phrase entity or the body of a Website."
+
+    originTypes = {'Phrase', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Phrase to Search for': {'description': 'Please enter the Phrase to be searched for. Note that the '
+                                                          'search is performed in the "Notes" field of a Phrase or the '
+                                                          'body of the selected Website entities, not their primary '
+                                                          'fields.',
+                                           'type': 'String',
+                                           'value': ''},
+                  'Case Sensitive': {'description': 'Do you want the phrase to be case sensitive?',
+                                     'type': 'SingleChoice',
+                                     'value': {'Yes', 'No'}
+                                     }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import re
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
+        }
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            offsets = []
+            counter = 0
+            searchPhrase = parameters['Phrase to Search for']
+            uid = entity['uid']
+            if entity['Entity Type'] == 'Phrase':
+                text = str(entity.get('Notes'))
+                primaryField = entity['Phrase']
+            else:  # Website entity
+                request = requests.get(entity["URL"], headers=headers)
+                primaryField = entity['URL']
+                text = request.text
+
+            if parameters['Case Sensitive'] == 'No':
+                iterator = re.finditer(rf"{searchPhrase}", text, re.IGNORECASE)
+                for match in iterator:
+                    offsets.append(match.start())
+                    counter += 1
+            elif parameters['Case Sensitive'] == 'Yes':
+                iterator = re.finditer(rf"{searchPhrase}", text)
+                for match in iterator:
+                    offsets.append(match.start())
+                    counter += 1
+
+            if counter > 0:
+                returnResults.append([{'Phrase': f'{primaryField} Contains Phrase: "{searchPhrase}" {counter} times',
+                                       'Entity Type': 'Phrase',
+                                       'Notes': f'"{searchPhrase}" was found {counter} time(s)\n'
+                                                f'Offsets: Matches at character indices: '
+                                                f'{(", ".join(map(str, offsets)))}'},
+                                      {uid: {'Resolution': f'Contains Phrase {searchPhrase}',
+                                             'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/CryptoAddressExtractor.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+
+"""
+Credit to @cyb_detective:
+https://medium.com/@cyb_detective/20-regular-expressions-examples-to-search-for-data-related-to-cryptocurrencies-43e31dd4a5dc
+"""
+
+
+class CryptoAddressExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Cryptocurrency Addresses"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns patterns matching common cryptocurrency address formats on a website."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Crypto Wallet'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import re
+
+        returnResults = []
+
+        ethRegex = re.compile(r"\b0[xX][a-fA-F0-9]{40}\b")
+        btcRegex = re.compile(
+            r"\b(?:bc(?:0(?:[ac-hj-np-z02-9]{39}|[ac-hj-np-z02-9]{59})|1[ac-hj-np-z02-9]{8,87})|[13][a-km-zA-HJ-NP-Z1-9]{25,34})\b")
+        bchRegex = re.compile(r"\b(?:(?:bitcoincash|bchreg|bchtest):)?[qp][a-z0-9]{41}\b")
+        moneroRegex = re.compile(r"\b[48][0-9AB][1-9A-HJ-NP-Za-km-z]{93}\b")
+        dogeRegex = re.compile(r"\bD[5-9A-HJ-NP-U][1-9A-HJ-NP-Za-km-z]{32}\b")
+        dashRegex = re.compile(r"\bX[1-9A-HJ-NP-Za-km-z]{33}\b")
+        rippleRegex = re.compile(r"\br[1-9A-HJ-NP-Za-km-z]{24,34}\b")
+        neoRegex = re.compile(r"\bN[0-9a-zA-Z]{33}\b")
+        litecoinRegex = re.compile(r"\b[LM3][a-km-zA-HJ-NP-Z1-9]{26,33}\b")
+        cosmosRegex = re.compile(r"\bcosmos[a-zA-Z0-9_.-]{10,}\b")
+        cardanoRegex = re.compile(r"\baddr1[a-z0-9]{10,}\b")
+        iotaRegex = re.compile(r"\biota[a-z0-9]{10,}\b")
+        liskRegex = re.compile(r"\b[0-9]{19}L\b")
+        nemRegex = re.compile(
+            r"\bN[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}\b")
+        ontologyRegex = re.compile(r"\bA[0-9a-zA-Z]{33}\b")
+        polkadotRegex = re.compile(r"\b1[0-9a-zA-Z]{47}\b")
+        stellarRegex = re.compile(r"\bG[0-9A-Z]{55}\b")  # Stellar addresses are always 56 characters long.
+
+        # The software can deduplicate, but handling it here is better.
+        allWallets = set()
+
+        def extractCryptoAddresses(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            # Remove <span> and <noscript> tags.
+            while True:
+                try:
+                    soupContents.noscript.extract()
+                except AttributeError:
+                    break
+            while True:
+                try:
+                    soupContents.span.extract()
+                except AttributeError:
+                    break
+            siteContent = soupContents.get_text()
+
+            ethMatch = ethRegex.findall(siteContent)
+            for potentialMatch in ethMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Etherium',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Etherium Wallet Address',
+                                                        'Notes': ''}}])
+            btcMatch = btcRegex.findall(siteContent)
+            for potentialMatch in btcMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Bitcoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Bitcoin or Bitcoin Cash Wallet Address',
+                                                        'Notes': ''}}])
+            bchMatch = bchRegex.findall(siteContent)
+            for potentialMatch in bchMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Bitcoin Cash',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Bitcoin Cash Wallet Address',
+                                                        'Notes': ''}}])
+            xmrMatch = moneroRegex.findall(siteContent)
+            for potentialMatch in xmrMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Monero',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Monero Wallet Address',
+                                                        'Notes': ''}}])
+            dogeMatch = dogeRegex.findall(siteContent)
+            for potentialMatch in dogeMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Dogecoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Dogecoin Wallet Address',
+                                                        'Notes': ''}}])
+            dashMatch = dashRegex.findall(siteContent)
+            for potentialMatch in dashMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Dash',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Dash Wallet Address',
+                                                        'Notes': ''}}])
+            rippleMatch = rippleRegex.findall(siteContent)
+            for potentialMatch in rippleMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Ripple',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Ripple Wallet Address',
+                                                        'Notes': ''}}])
+            neoMatch = neoRegex.findall(siteContent)
+            for potentialMatch in neoMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Neo',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Neo Wallet Address',
+                                                        'Notes': ''}}])
+            litecoinMatch = litecoinRegex.findall(siteContent)
+            for potentialMatch in litecoinMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Litecoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Litecoin Wallet Address',
+                                                        'Notes': ''}}])
+            cosmosMatch = cosmosRegex.findall(siteContent)
+            for potentialMatch in cosmosMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Cosmos',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Cosmos Wallet Address',
+                                                        'Notes': ''}}])
+            cardanoMatch = cardanoRegex.findall(siteContent)
+            for potentialMatch in cardanoMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Cardano',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Cardano Wallet Address',
+                                                        'Notes': ''}}])
+            iotaMatch = iotaRegex.findall(siteContent)
+            for potentialMatch in iotaMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Iota',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Iota Wallet Address',
+                                                        'Notes': ''}}])
+            liskMatch = liskRegex.findall(siteContent)
+            for potentialMatch in liskMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Lisk',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Lisk Wallet Address',
+                                                        'Notes': ''}}])
+            nemMatch = nemRegex.findall(siteContent)
+            for potentialMatch in nemMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Nem',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Nem Wallet Address',
+                                                        'Notes': ''}}])
+            ontologyMatch = ontologyRegex.findall(siteContent)
+            for potentialMatch in ontologyMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Ontology',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Ontology Wallet Address',
+                                                        'Notes': ''}}])
+            polkadotMatch = polkadotRegex.findall(siteContent)
+            for potentialMatch in polkadotMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Polkadot',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Polkadot Wallet Address',
+                                                        'Notes': ''}}])
+            stellarMatch = stellarRegex.findall(siteContent)
+            for potentialMatch in stellarMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Stellar',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Stellar Wallet Address',
+                                                        'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractCryptoAddresses(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/DecodePhrase.py

@@ -0,0 +1,56 @@
+class DecodePhrase:
+    # A string that is treated as the name of this resolution.
+    name = "Decode Phrase"
+
+    category = "String Operations"
+
+    # A string that describes this resolution.
+    description = "Decode encoded phrases."
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'Phrase'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Phrase'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {'Encoding Type': {'description': 'Select the type of encoding that was applied to the data.',
+                                    'type': 'SingleChoice',
+                                    'value': {'Binary', 'Hexadecimal', 'Base64'}
+                                    }}
+
+    def resolution(self, entityJsonList, parameters):
+        import base64
+        returnResult = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+
+            if parameters['Encoded Type'] == 'Hexadecimal':
+                text = entity[list(entity)[1]].strip()
+                data = str(bytearray.fromhex(text).decode())
+                returnResult.append([{'Phrase': data,
+                                      'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Hexadecimal Phrase', 'Notes': ''}}])
+
+            elif parameters['Encoded Type'] == 'Base64':
+                text = entity[list(entity)[1]].strip()
+                data = base64.b64decode(text).decode()
+                returnResult.append([{'Phrase': str(data),
+                                      'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Base 64 Decoded Phrase', 'Notes': ''}}])
+
+            elif parameters['Encoded Type'] == 'Binary':
+                text = entity[list(entity)[1]].replace(' ', '')
+                if len(text) % 8 != 0:
+                    return "Malformed format not in Octaves"
+
+                ascii_string = ''.join(chr(int(text[binaryIndex: binaryIndex + 8], 2))
+                                       for binaryIndex in range(0, len(text), 8))
+
+                returnResult.append([{'Phrase': ascii_string, 'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Binary Decoded Phrase', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/DeleteColumn.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+
+class DeleteColumn:
+    # A string that is treated as the name of this resolution.
+    name = "Rename or Delete Column"
+
+    category = "Spreadsheet Operations"
+
+    # A string that describes this resolution.
+    description = "Deletes the column with the specified index from a Spreadsheet document."
+
+    originTypes = {'Spreadsheet'}
+
+    resultTypes = {'Spreadsheet'}
+
+    parameters = {'Working Sheet': {'description': 'The name or index of the Sheet to read in the Spreadsheet '
+                                                   'file. By default, the first Sheet is used.',
+                                    'type': 'String',
+                                    'value': '0',
+                                    'default': '0'},
+                  'Column Name to Rename': {'description': 'Please enter the name of the column that you wish to  '
+                                                           'rename or delete.',
+                                            'type': 'String',
+                                            'value': ''},
+                  'New Column Name': {'description': 'Please enter the new name for the column.\nEnter the same name '
+                                                     'to delete the column instead.',
+                                      'type': 'String',
+                                      'value': ''}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import pandas as pd
+        import contextlib
+
+        workingSheet = parameters['Working Sheet']
+        with contextlib.suppress(ValueError):
+            workingSheet = int(workingSheet)
+
+        renameColumn = parameters['Column Name to Rename']
+        targetColumn = parameters['New Column Name']
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not filePath.exists() or not filePath.is_file():
+                continue
+
+            try:
+                csvDF = pd.read_excel(filePath, sheet_name=workingSheet)
+            except ValueError:
+                continue
+
+            if renameColumn == targetColumn:
+                csvDF.drop(renameColumn, inplace=True)
+            else:
+                csvDF.rename(columns={renameColumn: targetColumn}, inplace=True)
+
+            count = 0
+            while True:
+                newFileName = f"{filePath.name.split(filePath.suffix, 1)[0]}-c{count}{filePath.suffix}"
+                newFilePath = filePath.parent / newFileName
+                if not newFilePath.exists():
+                    break
+            csvDF.to_excel(newFilePath, index=False)
+
+            returnResults.append([{'Spreadsheet Name': newFileName,
+                                   'File Path': newFileName,
+                                   'Entity Type': 'Spreadsheet'},
+                                  {uid: {'Resolution': 'Rename/Delete Column',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/DomainFromPhrase.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+
+class DomainFromPhrase:
+    name = "Domain From Phrase"
+    category = "String Operations"
+    description = "Extract domain-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import contextlib
+        import tldextract
+
+        domainRegex = re.compile(
+            r'^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.'
+            r'([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$'
+        )
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase'].lower()
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if domainRegex.match(entityChunk):
+                    with contextlib.suppress(Exception):
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'Domain Name': entityChunk,
+                                                   'Entity Type': 'Domain'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Domain',
+                                                                   'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/EmailExtractor.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+
+class EmailExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Emails"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns the email addresses present on a website or index page of a domain."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Email Address'}
+
+    parameters = {'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
+                                               'code of the website. However, not all emails on the site may exist in '
+                                               'that format. Using Regex can result in more emails being extracted, '
+                                               'however it is possible that some false positives may be extracted '
+                                               'too.\nDo you want to also use Regex to extract emails, in addition to '
+                                               'the default extraction method?',
+                                'type': 'SingleChoice',
+                                'value': {'Yes', 'No'},
+                                'default': 'Yes'},
+                  'Verify Email Domain Validity': {'description': 'Verification checks are performed on extracted '
+                                                                  'emails to ensure that they are valid and working '
+                                                                  'email addresses. One of these checks involves '
+                                                                  'attempting to resolve the email address domain. '
+                                                                  'This will generate network traffic.\n'
+                                                                  'Do you want to verify email domain validity?',
+                                                   'type': 'SingleChoice',
+                                                   'value': {'Yes', 'No'},
+                                                   'default': 'No'}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import re
+        import contextlib
+        from email_validator import validate_email, caching_resolver, EmailNotValidError
+
+        returnResults = []
+
+        # Source: https://emailregex.com/
+        # Alt: (?:[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+(\.([a-zA-Z0-9-])+)+)
+        emailRegex = re.compile(
+            r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""")
+        useRegex = parameters['Use Regex'] == 'Yes'
+
+        resolver = caching_resolver(timeout=10)
+        verifyDomain = parameters['Verify Email Domain Validity'] == 'Yes'
+
+        # The software can deduplicate, but handling it here is better.
+        allEmails = set()
+
+        def extractEmails(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            if useRegex:
+                # Remove <span> and <noscript> tags, and attempt some basic de-obfuscation.
+                while True:
+                    try:
+                        soupContents.noscript.extract()
+                    except AttributeError:
+                        break
+                while True:
+                    try:
+                        soupContents.span.extract()
+                    except AttributeError:
+                        break
+                siteContent = soupContents.get_text()
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*at\s*(\]|\>|\))+\s*', '@', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*dot\s*(\]|\>|\))+\s*', '.', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*\.\s*(\]|\>|\))+\s*', '.', siteContent)
+
+                potentialEmails = emailRegex.findall(siteContent)
+                for potentialEmail in potentialEmails:
+                    with contextlib.suppress(EmailNotValidError):
+                        valid = validate_email(potentialEmail, dns_resolver=resolver, check_deliverability=verifyDomain)
+                        if valid.email not in allEmails:
+                            allEmails.add(valid.email)
+                            returnResults.append([{'Email Address': valid.email,
+                                                   'Entity Type': 'Email Address'},
+                                                  {currentUID: {'Resolution': 'Email Address Found',
+                                                                'Notes': ''}}])
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None and newLink.startswith('mailto:'):
+                    with contextlib.suppress(EmailNotValidError):
+                        valid = validate_email(newLink[7:], dns_resolver=resolver,
+                                               check_deliverability=verifyDomain)
+                        if valid.email not in allEmails:
+                            allEmails.add(valid.email)
+                            returnResults.append([{'Email Address': valid.email,
+                                                   'Entity Type': 'Email Address'},
+                                                  {currentUID: {'Resolution': 'Email Address Found',
+                                                                'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractEmails(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/EmailToDomain.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+class EmailToDomain:
+    name = "Email To Domain"
+    category = "Network Infrastructure"
+    description = "Get the Domain that an email address belongs to."
+    originTypes = {'Email Address'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            # There is no provider that I am aware of that allows '@' signs in the user part of the email.
+            with contextlib.suppress(Exception):
+                returnResults.append([{'Domain Name': primaryField.split('@')[1].strip(),
+                                       'Entity Type': 'Domain'},
+                                      {entity['uid']: {'Resolution': 'Email To Domain',
+                                                       'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/EmailToPhrase.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+
+class EmailToPhrase:
+    name = "Email Username To Phrase"
+    category = "String Operations"
+    description = "Get the username associated with the given email address."
+    originTypes = {'Email Address'}
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            returnResults.append([{'Phrase': primaryField.split('@')[0],
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'Email Username To Phrase',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ExtractDOCXMeta.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+
+class ExtractDOCXMeta:
+    # A string that is treated as the name of this resolution.
+    name = "Get DOCX Metadata"
+    category = "File Operations"
+
+    # A string that describes this resolution.
+    description = "Returns a set of nodes that contain all the metadata info of a docx file."
+
+    originTypes = {'Document', 'Archive'}
+
+    resultTypes = {'Phrase', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from docx2python import docx2python
+        import magic
+        from pathlib import Path
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+
+            if not filePath.exists() or not filePath.is_file():
+                continue
+
+            if magic.from_file(str(filePath), mime=True) != \
+                    'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+                continue  # We only care about docx files
+
+            metadata = docx2python(filePath)
+
+            data = metadata.properties
+            defaultDateProperties = ['created', 'modified']
+
+            modifiedDate = str(data.get('modified'))
+            modifiedDate = modifiedDate.replace('Z', '')
+            returnResults.append([{'Date': modifiedDate,
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'modified', 'Notes': ''}}])
+            createdDate = str(data.get('created'))
+            createdDate = createdDate.replace('Z', '')
+            returnResults.append([{'Date': createdDate,
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'created', 'Notes': ''}}])
+
+            returnResults.extend([{'Phrase': f'{metadataKey}: {str(data.get(metadataKey))}', 'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': metadataKey, 'Notes': ''}}] for metadataKey in
+                                 [dataKey for dataKey in data if dataKey not in defaultDateProperties])
+
+        return returnResults

Core/Resolutions/Core/ExtractDateCreated.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+
+class ExtractDateCreated:
+    name = "Extract Date Created"
+    category = "Date Operations"
+    description = "Extract the date in the Date Created field of the selected entities."
+    originTypes = {'*'}
+    resultTypes = {'Date'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        return_result = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            return_result.append([{'Date': str(entity['Date Created']),
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Extract Dates', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/ExtractPDFMeta.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+
+class ExtractPDFMeta:
+    # A string that is treated as the name of this resolution.
+    name = "Get PDF Metadata"
+    category = "File Operations"
+
+    # A string that describes this resolution.
+    description = "Returns a set of nodes that contain notable metadata info of pdf files."
+
+    originTypes = {'Document'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pypdf import PdfReader
+        from datetime import datetime, timedelta
+        import magic
+        from pathlib import Path
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+
+            if not (filePath.exists() and filePath.is_file()):
+                continue
+
+            if magic.from_file(str(filePath), mime=True) != 'application/pdf':
+                continue
+
+            with open(filePath, 'rb') as f:
+                pdf = PdfReader(f)
+                info = pdf.metadata
+                number_of_pages = len(pdf.pages)
+
+            for metadataKey in info:
+                if metadataKey.startswith('/'):
+                    attrValue = metadataKey[1:]
+                else:
+                    attrValue = metadataKey
+                if 'Date' in metadataKey:
+                    try:
+                        strDate = info[metadataKey].split(':', 1)[1]
+                        if strDate.endswith('Z'):
+                            dateString = datetime.strptime(strDate, "%Y%m%d%H%M%SZ").isoformat()
+                        elif '+' in strDate:
+                            datePart1, datePart2 = strDate.split('+', 1)
+                            date1 = datetime.strptime(datePart1, "%Y%m%d%H%M%S")
+                            date2 = timedelta(hours=int(datePart2.split("'")[0]), minutes=int(datePart2.split("'")[1]))
+                            dateString = (date1 + date2).isoformat()
+                        elif '-' in strDate:
+                            datePart1, datePart2 = strDate.split('-', 1)
+                            date1 = datetime.strptime(datePart1, "%Y%m%d%H%M%S")
+                            date2 = timedelta(hours=int(datePart2.split("'")[0]), minutes=int(datePart2.split("'")[1]))
+                            dateString = (date1 - date2).isoformat()
+                        else:
+                            raise ValueError('Cannot parse Date format.')
+
+                        returnResults.append([{'Date': dateString,
+                                               'Entity Type': 'Date'},
+                                              {uid: {'Resolution': attrValue, 'Notes': ''}}])
+                    except Exception:
+                        # Reset strDate to default value
+                        strDate = info[metadataKey]
+                        returnResults.append([{'Date': strDate,
+                                               'Entity Type': 'Date'},
+                                              {uid: {'Resolution': attrValue, 'Notes': ''}}])
+                else:
+                    # Clean some misshapen strings
+                    value = str(info[metadataKey])
+                    if value.startswith('/'):
+                        value = value[1:]
+
+                    returnResults.append([{'Phrase': f'{attrValue}: {value}',
+                                           'Entity Type': 'Phrase'},
+                                          {uid: {'Resolution': attrValue, 'Notes': ''}}])
+
+            returnResults.append([{'Phrase': f'Number of Pages: {number_of_pages}', 'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Number of Pages', 'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/FileExtractor.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+
+"""
+It should be noted that the effectiveness of this resolution relies on the website in question to have the non-website
+documents showing their extension in their links in the page. Since the extension is available the vast majority of the
+time, the resolution should produce correct results just about every time.
+
+The alternative is to analyze the contents of each page to determine if they are a non-web file, and then initiate a
+download. This would require a lot of work for very little payoff, which would be error-prone in and of itself, so it
+is not pursued.
+
+A few sites may also present issues when downloads are attempted by a user agent without javascript.
+Even if the download does not work however, the site containing the file will be represented as a node on the graph,
+so the user can download the file themselves if there was any issue.
+"""
+
+
+class FileExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Find Hosted File URLs"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of files in websites and domains."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Website', 'Document', 'Spreadsheet', 'Image', 'Video', 'Archive'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        import tldextract
+        import requests
+        from hashlib import md5
+        from binascii import hexlify
+        from pathlib import Path
+        from bs4 import BeautifulSoup
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        fileTypes = (".sxw", ".odt", ".odg", ".odp", ".docx", ".pptx", ".ppsx", ".doc", ".csv",
+                     ".ppt", ".pps", ".pdf", ".wpd", ".raw", ".cr2", ".crw", ".indd", ".rdp", ".ica", ".ico", ".txt",
+                     ".text", ".bak", ".log", ".env", ".pub", ".docm", ".old", ".apk", ".sql", ".cfg",
+                     ".key", ".reg", ".yml", ".yaml", ".mail", ".eml", ".mbox", ".mbx", ".url", ".csr", ".config",
+                     ".mdb", ".user", ".adr", ".ini", ".plist", ".conf", ".dat", ".pcf", ".bok", ".properties", ".json",
+                     ".backup", ".sh", ".py", ".md", ".inc", '.ovpn', '.bat')
+        spreadsheetTypes = (".xlsx", ".xls", ".ods", ".xlsm")
+        videoTypes = (".mp3", ".mp4", ".mov", ".webm", ".amv")
+        imageTypes = (".jpg", ".jpeg", ".png", ".svg", ".svgz", ".bmp")
+        archiveTypes = (".zip", ".rar", ".7z", ".gz")
+
+        returnResults = []
+
+        def iterateOnDepth(currentURL, currentDepth: int):
+            urlsExplored.add(currentURL)
+            urlsToExplore = set()
+
+            for _ in range(3):
+                try:
+                    page.goto(currentURL, wait_until="networkidle", timeout=10000)
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            urlInPage = soupContents.find_all('a') + soupContents.find_all('link')
+            for tag in urlInPage:
+                link = tag.get('href', None)
+                if link is not None:
+                    if not link.startswith('http'):
+                        # We assume that we will be redirected to https if available.
+                        link = f'http://{domain}{link}'
+                    link = link.split('#')[0]
+                    if link not in urlsExplored:
+                        urlsExplored.add(link)
+
+                        fileTypeIdentified = ''
+
+                        # Material name is the part after the last slash of the URL, plus the sha512sum of the URL
+                        if link.endswith(fileTypes):
+                            fileTypeIdentified = 'Document'
+                        elif link.endswith(videoTypes):
+                            fileTypeIdentified = 'Video'
+                        elif link.endswith(imageTypes):
+                            fileTypeIdentified = 'Image'
+                        elif link.endswith(archiveTypes):
+                            fileTypeIdentified = 'Archive'
+                        elif link.endswith(spreadsheetTypes):
+                            fileTypeIdentified = 'Spreadsheet'
+
+                        if fileTypeIdentified:
+                            childIndex = len(returnResults)
+
+                            returnResults.append([{'URL': link,
+                                                   'Entity Type': 'Website'},
+                                                  {uid: {'Resolution': 'File URL',
+                                                         'Notes': ''}}])
+
+                            docProperName = link.split('/')[-1]
+                            docFileName = f'{hexlify(md5(link.encode()).digest()).decode()} | {docProperName}'
+                            docFullPath = Path(parameters['Project Files Directory']) / docFileName
+
+                            with contextlib.suppress(Exception):
+                                response = requests.get(link,
+                                                        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+                                                                               'x64; rv:94.0) Gecko/20100101 '
+                                                                               'Firefox/94.0'},
+                                                        stream=True)
+                                with open(docFullPath, 'wb') as fileToWrite:
+                                    for chunk in response.iter_content(4096):
+                                        fileToWrite.write(chunk)
+
+                                returnResults.append([{f'{fileTypeIdentified} Name': docProperName,
+                                                       'File Path': docFileName,
+                                                       'Entity Type': fileTypeIdentified},
+                                                      {childIndex: {'Resolution': 'Downloaded File', 'Notes': ''}}])
+
+                        elif domain in link:
+                            urlsToExplore.add(link)
+
+            linksInImgSrc = soupContents.find_all('img')
+            for tag in linksInImgSrc:
+                link = tag.get('src', None)
+                if link is not None:
+                    if not link.startswith('http'):
+                        # We assume that we will be redirected to https if available.
+                        link = f'http://{domain}{link}'
+                    link = link.split('#')[0]
+                    if link not in urlsExplored:
+                        urlsExplored.add(link)
+
+                        childIndex = len(returnResults)
+
+                        returnResults.append([{'URL': link,
+                                               'Entity Type': 'Website'},
+                                              {uid: {'Resolution': 'File URL',
+                                                     'Notes': ''}}])
+                        docProperName = link.split('/')[-1]
+                        docFileName = f'{hexlify(md5(link.encode()).digest()).decode()} | {docProperName}'
+                        docFullPath = Path(parameters['Project Files Directory']) / docFileName
+
+                        with contextlib.suppress(Exception):
+                            response = requests.get(link,
+                                                    headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+                                                                           'x64; rv:94.0) Gecko/20100101 '
+                                                                           'Firefox/94.0'},
+                                                    stream=True)
+                            with open(docFullPath, 'wb') as fileToWrite:
+                                for chunk in response.iter_content(4096):
+                                    fileToWrite.write(chunk)
+
+                            returnResults.append([{'Image Name': docProperName,
+                                                   'File Path': docFileName,
+                                                   'Entity Type': 'Image'},
+                                                  {childIndex: {'Resolution': 'Downloaded File',
+                                                                'Notes': ''}}])
+            if currentDepth > 0:
+                newDepth = currentDepth - 1
+                for newURL in urlsToExplore:
+                    iterateOnDepth(newURL, newDepth)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                domain = tldextract.extract(url).fqdn
+
+                # Because these do not persist across entities, it is possible to explore a URL multiple times.
+                # However, since different URLs may be encountered at different depths, this way should ensure
+                #   that there are no false negatives, i.e. if something should be discovered, it will be.
+                urlsExplored = set()
+                iterateOnDepth(url, maxDepth)
+
+            return returnResults

Core/Resolutions/Core/FileHasher.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+
+class FileHasher:
+    name = "Get File Hash"
+    category = "File Operations"
+    description = "Get the Hash of a file."
+    originTypes = {"Image", "Document", "Spreadsheet", "Video", "Archive", "Disk"}
+    resultTypes = {'Hash'}
+    parameters = {'hashing_algorithms': {'description': 'The type of hash/es that will be returned',
+                                         'type': 'MultiChoice',
+                                         'value': {'SHA1', 'SHA256', 'MD5'}
+                                         }}
+
+    def resolution(self, entityJsonList, parameters):
+        import hashlib
+        from pathlib import Path
+
+        return_result = []
+        hashing_algorithms = parameters['hashing_algorithms']
+        for entity in entityJsonList:
+            uid = entity['uid']
+            file_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not (file_path.exists() and file_path.is_file()):
+                continue
+            block_size = 65536  # The size of each read from the file
+            for hashing_algorithm in hashing_algorithms:
+                if hashing_algorithm == "SHA1":
+                    file_hash = hashlib.sha1()  # nosec
+                elif hashing_algorithm == "SHA256":
+                    file_hash = hashlib.sha256()  # nosec
+                else:
+                    file_hash = hashlib.md5()  # nosec
+                with open(file_path, 'rb') as f:
+                    fb = f.read(block_size)
+                    while len(fb) > 0:
+                        file_hash.update(fb)
+                        fb = f.read(block_size)
+                resulting_hash = file_hash.hexdigest()
+                return_result.append([{'Hash Value': resulting_hash,
+                                       'Hash Algorithm': hashing_algorithm,
+                                       'Entity Type': 'Hash'},
+                                      {uid: {'Resolution': f'{hashing_algorithm} Hash', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/GetExternalURLs.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+"""
+This resolution can be notoriously hard to get working. Because one cannot get any indication of when the scripts on
+a page have finished running, it is possible that some results are missed if one is not extremely careful.
+
+Waiting until network traffic has ended should be the safest way to ensure that all javascript has loaded, and as such,
+it would be the most likely point in time where the scripts on the visited page have exposed any useful information
+to be gleamed.
+
+Of course, that runs the risk of the operation timing out for whatever reason, or the operation taking a very long
+time to complete. It also means that making it async risks that some information is missed due to errors, since
+catching them would essentially negate the benefit of doing operations asynchronously.
+
+Tweaks to this script can be made depending on each investigator's workflow.
+"""
+
+
+class GetExternalURLs:
+    name = "Get External Urls"
+    category = "Website Information"
+    description = "Returns all links to external sites on a website."
+    originTypes = {'Website'}
+    resultTypes = {'Website', 'Onion Website'}
+
+    parameters = {'Element types to check': {'description': 'Select the types of elements to investigate for '
+                                                            'external links. Note that "a" elements have the lowest '
+                                                            'chance to present false positives, whereas other types '
+                                                            'of elements might point to websites that are affiliated '
+                                                            'with the input URL in some way, such as content delivery '
+                                                            'networks owned by the same entity.',
+                                             'type': 'MultiChoice',
+                                             'value': {'<a> elements', '<img> elements', '<link> elements'},
+                                             'default': ['<a> elements']
+                                             }}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import re
+        import urllib.parse
+
+        onionRegex = re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$""")
+        returnResult = []
+
+        extract_a = '<a> elements' in parameters['Element types to check']
+        extract_img = '<img> elements' in parameters['Element types to check']
+        extract_link = '<link> elements' in parameters['Element types to check']
+
+        # Sites like youtube replace external links with a redirect link originating
+        #   from the site itself. This sort of gets around that.
+        redirectRegex = re.compile(r'\?.*(q|url)=\S[^&#?]+', re.IGNORECASE)
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+            externalUrls = {}
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                parsedURL = urllib.parse.urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+                domain = tldextract.extract(url).fqdn
+
+                # Try to load the page a few times, in case of timeouts.
+                # I don't think making parts of this async actually helps in this case.
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                        if extract_a:
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]):
+                                    if domain in link:
+                                        redirectLinks = redirectRegex.findall(link)
+                                        if 'redirect' in link and len(redirectLinks) > 0:
+                                            try:
+                                                newLink = str(urllib.parse.unquote(redirectLinks[0]))[2:]
+                                                try:
+                                                    externalUrls[newLink].add(uid)
+                                                except KeyError:
+                                                    externalUrls[newLink] = {uid}
+                                            except IndexError:
+                                                try:
+                                                    externalUrls[redirectLinks[0]].add(uid)
+                                                except KeyError:
+                                                    externalUrls[redirectLinks[0]] = {uid}
+                                            except Exception:
+                                                pass
+
+                                    else:
+                                        newLink = link.split('#')[0].split('?')[0]
+                                        try:
+                                            externalUrls[newLink].add(uid)
+                                        except KeyError:
+                                            externalUrls[newLink] = {uid}
+                        if extract_img:
+                            linksInImgSrc = soupContents.find_all('img')
+                            for tag in linksInImgSrc:
+                                link = tag.get('src', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
+                                    newLink = link.split('#')[0].split('?')[0]
+                                    try:
+                                        externalUrls[newLink].add(uid)
+                                    except KeyError:
+                                        externalUrls[newLink] = {uid}
+
+                        if extract_link:
+                            linksInLinkHref = soupContents.find_all('link')
+                            for tag in linksInLinkHref:
+                                link = tag.get('href', None)
+                                parsedURL = urllib.parse.urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
+                                    newLink = link.split('#')[0].split('?')[0]
+                                    try:
+                                        externalUrls[newLink].add(uid)
+                                    except KeyError:
+                                        externalUrls[newLink] = {uid}
+                        break
+                    except TimeoutError:
+                        pass
+                    except Error:
+                        break
+
+            page.close()
+            context.close()
+            browser.close()
+
+        for externalUrl in externalUrls:
+            onionCheck = onionRegex.findall(externalUrl)
+            for urlUid in externalUrls[externalUrl]:
+                if len(onionCheck) == 1:
+                    returnResult.append([{'Onion URL': externalUrl, 'Entity Type': 'Onion Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+                else:
+                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/GetInternalURLs.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+
+class GetInternalURLs:
+    name = "Get Domain Urls"
+    category = "Website Information"
+    description = "Returns all URLs in a Domain."
+    originTypes = {'Website', 'Domain'}
+    resultTypes = {'Website'}
+
+    parameters = {'Include Resources': {'description': 'Should the resolution also return links to images and internal '
+                                                       'links (i.e. links to resources used by the site)? Note that '
+                                                       'doing so will make the resolution take more time to complete.',
+                                        'type': 'SingleChoice',
+                                        'value': {'Include images and internal links', 'Only consider links to pages'},
+                                        'default': 'Only consider links to pages'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import urllib.parse
+
+        returnResult = []
+        internalUrls = {}
+
+        considerResources = False if parameters['Include Resources'] == 'Only consider links to pages' else True
+
+        def handleLink(currentLink, currentUrl, currentDomain):
+            if currentLink is None:
+                return None
+            if currentLink.startswith('//'):
+                if currentUrl.endswith('/'):
+                    currentUrl = currentUrl[:-1]
+                currentLink = currentUrl + currentLink[1:]
+            elif currentLink.startswith('/'):
+                urlParts = urllib.parse.urlparse(currentUrl)
+                currentLink = urlParts.scheme + '://' + urlParts.netloc + currentLink
+            parsedCurrentURL = urllib.parse.urlparse(currentLink)
+            if all([parsedCurrentURL.scheme, parsedCurrentURL.netloc]):
+                if currentDomain in currentLink:
+                    newLink = currentLink.split('#')[0].split('?')[0]
+                    if newLink.endswith('/'):
+                        newLink = newLink[:-1]
+                    return newLink
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                entityType = site['Entity Type']
+                if entityType == 'Domain':
+                    url = 'http://' + site['Domain Name']
+                else:
+                    url = site['URL']
+                parsedURL = urllib.parse.urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+                domain = tldextract.extract(url).fqdn
+
+                domainUrls = [url]
+                index = 0
+                while index < len(domainUrls):
+                    url = domainUrls[index]
+                    index += 1
+                    for _ in range(3):
+                        try:
+                            page.goto(url, wait_until="networkidle", timeout=10000)
+                            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                            if domain not in page.url:
+                                # Handle redirects to other sites
+                                try:
+                                    internalUrls[page.url].add(uid)
+                                except KeyError:
+                                    internalUrls[page.url] = {uid}
+                                break
+
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                handledLink = handleLink(link, url, domain)
+                                if handledLink:
+                                    try:
+                                        internalUrls[handledLink].add(uid)
+                                    except KeyError:
+                                        internalUrls[handledLink] = {uid}
+                                    if handledLink not in domainUrls:
+                                        domainUrls.append(handledLink)
+
+                            if considerResources:
+                                linksInImgSrc = soupContents.find_all('img')
+                                for tag in linksInImgSrc:
+                                    link = tag.get('src', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+
+                                linksInLinkHref = soupContents.find_all('link')
+                                for tag in linksInLinkHref:
+                                    link = tag.get('href', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+                            break
+                        except TimeoutError:
+                            pass
+                        except Error:
+                            break
+
+            page.close()
+            context.close()
+            browser.close()
+
+        for internalUrl in internalUrls:
+            for urlUid in internalUrls[internalUrl]:
+                returnResult.append([{'URL': internalUrl, 'Entity Type': 'Website'},
+                                     {urlUid: {'Resolution': 'Internal Link', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/GetWebsiteText.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+
+class GetWebsiteText:
+    # A string that is treated as the name of this resolution.
+    name = "Get Website Text"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns the text content of the selected websites."
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from bs4 import BeautifulSoup
+        from bs4.element import Comment
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from urllib.parse import urlparse
+
+        returnResults = []
+
+        def tag_visible(element):
+            if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
+                return False
+            return not isinstance(element, Comment)
+
+        def text_from_html(body):
+            soup = BeautifulSoup(body, 'lxml')
+            texts = soup.findAll(text=True)
+            visible_texts = filter(tag_visible, texts)
+            return u" ".join(t.strip() for t in visible_texts if t.strip() != '')
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                parsedURL = urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+
+                # Try to load the page a few times, in case of timeouts.
+                # I don't think making parts of this async actually helps in this case.
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        textContent = text_from_html(page.content())
+                        returnResults.append([{'Phrase': f'Website Body of: {url}',
+                                               'Notes': textContent,
+                                               'Entity Type': 'Phrase'},
+                                              {uid: {'Resolution': 'Website Body', 'Notes': ''}}])
+                        break
+                    except TimeoutError:
+                        pass
+                    except Error:
+                        break
+
+        return returnResults

Core/Resolutions/Core/HostnameToDomain.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+class HostnameToDomain:
+    name = "Get Top Level Domain"
+    category = "Network Infrastructure"
+    description = "Get the top level domain of any given website or subdomain."
+    originTypes = {'Domain', 'Website'}
+    resultTypes = {'Domain'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from tldextract import extract
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]].strip()
+            tsd, td, tsu = extract(primary_field)
+            domain = f'{td}.{tsu}'
+            if domain == primary_field:
+                continue
+            return_result.append([{'Domain Name': domain,
+                                   'Entity Type': 'Domain'},
+                                  {uid: {'Resolution': 'Domain to Hostname', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/HostnameToIP.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+
+class HostnameToIP:
+    name = "Hostname To IP"
+    category = "Network Infrastructure"
+    description = "Gets the IP associated with the given hostname."
+    originTypes = {'Domain', 'Website'}
+    resultTypes = {'IP Address', 'IPv6 Address'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import socket
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            urlFeatures = primaryField.find('://')
+            if urlFeatures != -1:
+                primaryField = primaryField[urlFeatures + 3:]
+            primaryField = primaryField.split('/')[0]  # Remove trailing slashes
+            try:
+                entityResults = socket.getaddrinfo(primaryField, 443, proto=socket.IPPROTO_TCP) + \
+                                socket.getaddrinfo(primaryField, 80, proto=socket.IPPROTO_TCP)
+            except Exception:
+                continue
+
+            for result in entityResults:
+                if result[0].value == 2:
+                    returnResults.append([{'IP Address': result[4][0],
+                                           'Entity Type': 'IP Address'},
+                                          {entity['uid']: {'Resolution': 'Hostname To IP',
+                                                           'Notes': ''}}])
+                elif result[0].value == 10:
+                    returnResults.append([{'IPv6 Address': result[4][0],
+                                           'Entity Type': 'IPv6 Address'},
+                                          {entity['uid']: {'Resolution': 'Hostname To IP',
+                                                           'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/IPToASN.py

@@ -0,0 +1,53 @@
+class IPToASN:
+    # A string that is treated as the name of this resolution.
+    name = "Get ASN From IP"
+
+    category = "Network Infrastructure"
+
+    # A string that describes this resolution.
+    description = "Get the Autonomous System Number that the selected IP Addresses belong to."
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'IP Address'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Autonomous System'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois.net import Net
+        from ipwhois.asn import IPASN
+        import pycountry
+
+        returnResult = []
+
+        for entity in entityJsonList:
+            if entity['Entity Type'] == 'IP Address':
+                uid = entity['uid']
+                IP_add = entity[list(entity)[1]]
+                net = Net(IP_add)
+                obj = IPASN(net)
+                results = obj.lookup()
+                index_of_child = len(returnResult)
+                countryCode = results['asn_country_code']
+                country = pycountry.countries.get(alpha_2=countryCode).name
+                returnResult.extend(([{'AS Number': "AS" + results['asn'],
+                                       'ASN Cidr': results['asn_cidr'],
+                                       'Date Created': results['asn_date'],
+                                       'Entity Type': 'Autonomous System'},
+                                      {uid: {'Resolution': 'Autonomous System of IP', 'Notes': ''}}],
+                                     [{'Organization Name': results['asn_registry'],
+                                       'Entity Type': 'Organization'},
+                                      {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}],
+                                     [{'Country Name': country,
+                                       'Entity Type': 'Country'},
+                                      {index_of_child: {'Resolution': 'Country of Registry for ASN', 'Notes': ''}}],
+                                     [{'Phrase': results['asn_description'],
+                                       'Entity Type': 'Phrase'},
+                                      {index_of_child: {'Resolution': 'ASN Description', 'Notes': ''}}]))
+
+        return returnResult

Core/Resolutions/Core/IPToHostname.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+
+class IPToHostname:
+    name = "IP To Hostname"
+    category = "Network Infrastructure"
+    description = "Gets the Fully Qualified Domain Name associated with the given IPv4 / IPv6 address."
+    originTypes = {'IPv6 Address', 'IP Address'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import socket
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            try:
+                fqdn = socket.getfqdn(primaryField)
+            except Exception:
+                continue
+
+            if fqdn != primaryField:
+                returnResults.append([{'Domain Name': fqdn,
+                                       'Entity Type': 'Domain'},
+                                      {entity['uid']: {'Resolution': 'Fully Qualified Domain Name',
+                                                       'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/IPWhois.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+
+class IPWhois:
+    name = "IPv4 WhoIs Information"
+    category = "Network Infrastructure"
+    description = "Extract information from the Whois record that corresponds to a given IP Address."
+    originTypes = {'IP Address'}
+    resultTypes = {'Country', 'Autonomous System', 'Email Address'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois import IPWhois
+        from ipaddress import ip_address
+        import pycountry
+
+        return_result = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]].strip()
+            try:
+                ip_address(primary_field)
+            except ValueError:
+                return "The Entity Provided isn't a valid IP Address"
+            IPobject = IPWhois(primary_field)
+            response = IPobject.lookup_whois()
+            return_result.append([{'AS Number': 'AS' + str(response['asn']),
+                                   'ASN Cidr': str(response['asn_cidr']),
+                                   'Entity Type': 'Autonomous System'},
+                                  {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+            for net in response['nets']:
+                if net['country'] is not None:
+                    if country := pycountry.countries.get(alpha_2=net['country']):
+                        country_name = country.name
+                    else:
+                        # May not always be an actual Country.
+                        country_name = net['country']
+                    return_result.append([{'Country Name': country_name,
+                                           'Entity Type': 'Country'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+                if net['name'] is not None:
+                    return_result.append([{'Company Name': net['name'],
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+                if net['emails'] is not None:
+                    return_result.extend([{'Email Address': email, 'Entity Type': 'Email Address'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}] for email in net['emails'])
+        return return_result

Core/Resolutions/Core/ImageToDevice.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+
+class ImageToDevice:
+    name = "Device From Image"
+    category = "File Operations"
+    description = "Find information about on what device an image was taken with."
+    originTypes = {"Image"}
+    resultTypes = {'Phrase'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from exif import Image
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            index_of_child = len(return_result)
+            image_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not image_path.exists() or not image_path.is_file():
+                continue
+            with open(image_path, 'rb') as image_file:
+                my_image = Image(image_file)
+                if my_image.has_exif is False:
+                    continue
+                for tag in my_image.list_all():
+                    if tag == "make":
+                        return_result.append([{'Phrase': my_image.make,
+                                               'Entity Type': 'Phrase'},
+                                              {uid: {'Resolution': 'ExifMetadata Device Manufacturer',
+                                                     'Notes': ''}}])
+                    elif tag == "model":
+                        return_result.append([{'Phrase': my_image.model,
+                                               'Entity Type': 'Phrase'},
+                                              {index_of_child: {'Resolution': 'ExifMetadata Device Model',
+                                                                'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/ImageToGeoLocation.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+
+class ImageToGeoLocation:
+    name = "Geolocation From Image"
+    category = "File Operations"
+    description = "Find information about where an image was taken."
+    originTypes = {"Image"}
+    resultTypes = {'GeoCoordinates'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from exif import Image
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            image_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not image_path.exists() or not image_path.is_file():
+                continue
+            with open(image_path, 'rb') as image_file:
+                my_image = Image(image_file)
+                if my_image.has_exif is False:
+                    continue
+                return_result.extend([{'Label': f"Location of {str(entity[list(entity)[1]].strip())}",
+                                       'Latitude': my_image.gps_latitude,
+                                       'Longitude': my_image.gps_longitude,
+                                       'Entity Type': 'GeoCoordinates'},
+                                      {uid: {'Resolution': 'GeoCoordinates', 'Notes': ''}}]
+                                     for tag in my_image.list_all() if tag == "gps_latitude")
+
+        return return_result

Core/Resolutions/Core/JSCodeExtractor.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+
+class JSCodeExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Tracking Codes"
+
+    category = "Website Tracking"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of 'ca-pub', 'ua' and 'gtm' tracking codes for websites and/or domains."
+
+    originTypes = {'Website', 'Domain'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, Error
+        from base64 import b64decode
+        import re
+        import contextlib
+
+        returnResults = []
+        requestUrlsParsed = set()
+
+        uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
+        pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
+        gtmRegex = re.compile(r'\bGTM-[A-Z\d]{1,7}\b')
+        gRegex = re.compile(r'\bG-[A-Z\d]{1,15}\b', re.IGNORECASE)
+        qualtricsRegex = re.compile(r'\bQ_(?:Z|S)ID=\w*\b')
+        pingdomRegex = re.compile(r'\bpa-[a-fA-F\d]{24}.js\b$')
+        mPulseRegex = re.compile(r'go-mpulse.net/boomerang/[A-Z\d]{5}(?:-[A-Z\d]{5}){4}\b')
+        contextWebRegex = re.compile(r'\.contextweb\.com.*token=.*')
+        facebookRegex = re.compile(r'facebook.com/tr?.*id=\d*')
+        googleMapsRegex = re.compile(r'maps\.googleapis\.com/maps/api/js\?.*client=[ a-zA-Z\d-]*')
+        marketoRegex = re.compile(r'marketo\.com/rtp-api/v1/rtp.js\?.*aid=[ a-zA-Z\d-]*')
+        visualWebsiteOptimizerRegex = re.compile(r'visualwebsiteoptimizer\.com/j\.php\?.*a=\d*')
+        optimizeRegex = re.compile(r'googleoptimize\.com/optimize\.js\?.*id=[A-Z\d-]*')
+        markMonitorRegex = re.compile(r'\.adsrvr\.org/track/evnt/\?.*adv=[a-zA-Z\d-]*')
+        zendeskRegex = re.compile(r'\.zdassets\.com/ekr/snippet\.js\?.*key=[a-zA-Z\d-]*')
+        quantServeRegex = re.compile(r'pixel\.quantserve\.com/pixel/.*\.gif\?')
+        cookieLawRegex = re.compile(r'cdn\.cookielaw\.org/consent/.*/')
+        oneTagRegex = re.compile(r'get\.s-onetag\.com/.*/')
+        bounceExchangeRegex = re.compile(r'tag\.bounceexchange\.com/.*/')
+        pushlyRegex = re.compile(r'cdn.p-n.io/.*domain_key=[\w%]*')
+        akamaiRegex = re.compile(r'/akam/.*a=[\w%=]*')
+        demdexRegex = re.compile(r'dpm\.demdex\.net/id\?.*d_orgid=[^&]*')
+        brightcoveRegex = re.compile(r'metrics\.brightcove\.com/.*/tracker\?.*&account=[^&]*')
+
+        def GetTrackingCodes(pageUid, requestUrl) -> None:
+            if requestUrl in requestUrlsParsed:
+                return
+            requestUrlsParsed.add(requestUrl)
+            for uaCode in uaRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': uaCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google UA Tracking Code',
+                                                 'Notes': ''}}])
+            for pubCode in pubRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pubCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google AdSense ca-pub Tracking Code',
+                                                 'Notes': ''}}])
+            for gtmCode in gtmRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': gtmCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google GTM Tracking Code',
+                                                 'Notes': ''}}])
+            for gCode in gRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': gCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google G Tracking Code',
+                                                 'Notes': ''}}])
+            for qCode in qualtricsRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': qCode[6:],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Qualtrics Tracking Code',
+                                                 'Notes': ''}}])
+            for pCode in pingdomRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pCode[:-3],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Pingdom Tracking Code',
+                                                 'Notes': ''}}])
+            for mCode in mPulseRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mCode.split('/')[-1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'mPulse Tracking Code',
+                                                 'Notes': ''}}])
+            for cCode in contextWebRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': cCode.split('token=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'ContextWeb Tracking Code',
+                                                 'Notes': ''}}])
+            for fCode in facebookRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': fCode.split('id=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Facebook Tracking Pixel Code',
+                                                 'Notes': ''}}])
+            for mapsCode in googleMapsRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mapsCode.split('client=', 1)[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google Maps Client Code',
+                                                 'Notes': ''}}])
+            for marketoCode in marketoRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': marketoCode.split('aid=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Marketo Tracking Code',
+                                                 'Notes': ''}}])
+            for vwoCode in visualWebsiteOptimizerRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': vwoCode.split('a=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Visual Website Optimizer Tracking User ID',
+                                                 'Notes': ''}}])
+            for oCode in optimizeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': oCode.split('id=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google Optimize ID',
+                                                 'Notes': ''}}])
+            for mmCode in markMonitorRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mmCode.split('adv=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Mark Monitor Tracking ID',
+                                                 'Notes': ''}}])
+            for zCode in zendeskRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': zCode.split('key=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Zendesk ID',
+                                                 'Notes': ''}}])
+            for qsCode in quantServeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': qsCode.split('/pixel/')[1].split('.gif')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'QuantServe Tracking Pixel ID',
+                                                 'Notes': ''}}])
+            for clCode in cookieLawRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': clCode.split('/')[2],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'CookieLaw Website ID',
+                                                 'Notes': ''}}])
+            for otCode in oneTagRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': otCode.split('/')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'OneTag Tracking ID',
+                                                 'Notes': ''}}])
+            for beCode in bounceExchangeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': beCode.split('/')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'BounceExchange Tracking ID',
+                                                 'Notes': ''}}])
+            for pushlyCode in pushlyRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pushlyCode.split('domain_key=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Pushly Website ID',
+                                                 'Notes': ''}}])
+            for aCode in akamaiRegex.findall(requestUrl):
+                encodedTracking = aCode.split('a=', 1)[1]
+                encodedTracking = encodedTracking.replace('%3D', '=')
+                decodedTracking = b64decode(encodedTracking).decode('utf-8').split('t=')[1].split('&')[0]
+                returnResults.append([{'Phrase': decodedTracking,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Akamai Website ID',
+                                                 'Notes': 'SHA-1 Sum'}}])
+            for dCode in demdexRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': dCode.split('d_orgid=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'DemDex (Adobe) Website ID',
+                                                 'Notes': ''}}])
+            for bCode in brightcoveRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': bCode.split('account=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'BrightCove Website ID',
+                                                 'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'
+            )
+            contextNoJS = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
+                java_script_enabled=False
+            )
+            pageJS = context.new_page()
+            pageNoJS = contextNoJS.new_page()
+            uid = None
+
+            # Subscribe to "request" events.
+            pageJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+            pageNoJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+
+                with contextlib.suppress(Error):
+                    pageJS.goto(url, wait_until="networkidle")
+                with contextlib.suppress(Error):
+                    pageNoJS.goto(url, wait_until="networkidle")
+            pageJS.close()
+            pageNoJS.close()
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/LongANStringExtractor.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+"""
+Credit to @cyb_detective:
+https://medium.com/@cyb_detective/20-regular-expressions-examples-to-search-for-data-related-to-cryptocurrencies-43e31dd4a5dc
+"""
+
+
+class LongANStringExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Long Alphanumeric Strings"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns patterns matching common cryptocurrency address formats on a website."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Minimum Length': {'description': 'Specify the minimum length an alphanumeric string has to have '
+                                                    'to be extracted.',
+                                     'type': 'String',
+                                     'value': '',
+                                     'default': '20'
+                                     }}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import re
+
+        try:
+            minLength = int(parameters['Minimum Length'])
+            if minLength < 1:
+                raise ValueError('Invalid min length specified.')
+        except ValueError:
+            return "Invalid Minimum Length specified."
+
+        returnResults = []
+
+        matchPattern = re.compile(r"\b[a-zA-Z0-9_.-]{" + str(minLength) + r",}\b")
+
+        # The software can deduplicate, but handling it here is better.
+        allPatterns = set()
+
+        def extractStrings(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            # Remove <span> and <noscript> tags.
+            while True:
+                try:
+                    soupContents.noscript.extract()
+                except AttributeError:
+                    break
+            while True:
+                try:
+                    soupContents.span.extract()
+                except AttributeError:
+                    break
+            siteContent = soupContents.get_text()
+
+            stringMatches = matchPattern.findall(siteContent)
+            for potentialMatch in stringMatches:
+                if potentialMatch not in allPatterns:
+                    allPatterns.add(potentialMatch)
+                    returnResults.append([{'Phrase': potentialMatch,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Long alphanumeric string',
+                                                        'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractStrings(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/NPMJSSearch.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+
+class NPMJSSearch:
+    name = "Find NPM organization"
+    category = "Online Identity"
+    description = "Find a collective's npmjs organization page."
+    originTypes = {'Phrase', 'Company', 'Organization'}
+    resultTypes = {'Website'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+
+        headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0'}
+        url_base = 'https://www.npmjs.com/org/'
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]].lower()
+
+            string_checks = set()
+            string_checks.add(''.join(primaryField.split(' ')))
+            string_checks.add('_'.join(primaryField.split(' ')))
+            string_checks.add('-'.join(primaryField.split(' ')))
+
+            for check in string_checks:
+                check_url = url_base + check
+                request = requests.head(check_url, headers=headers)
+                if request.status_code == 200:
+                    returnResults.append([{'URL': check_url,
+                                           'Entity Type': 'Website'},
+                                          {entity['uid']: {'Resolution': 'NPMJS Org',
+                                                           'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/PhoneNumbersExtractor.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+"""
+This extracts only phone numbers that are marked as a 'tel:' hyperlink.
+Extraction of other phone numbers is imprecise, as random numbers may be interpreted as valid phone numbers.
+"""
+
+
+class PhoneNumbersExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Phone Numbers"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns the Phone Numbers present on a website or index page of a domain."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phone Number'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        import re
+
+        cleanTagsRegex = re.compile(r'<.*?>')
+        phoneNumCharsExclusion = re.compile(r'[^ -+()\[\]\d]')
+
+        returnResults = []
+
+        def extractTels(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None and newLink.startswith('tel:'):
+                    returnResults.append([{'Phone Number': newLink[4:],
+                                           'Entity Type': 'Phone Number'},
+                                          {currentUID: {'Resolution': 'Phone Number Found',
+                                                        'Notes': ''}}])
+
+            textTags = soupContents.find_all('p')
+            for tag in textTags:
+                tagContents = tag.text
+                cleanTagContentsList = re.sub(cleanTagsRegex, '', tagContents).split('\n')
+                for cleanTag in cleanTagContentsList:
+                    cleanTag = cleanTag.strip()
+                    if not phoneNumCharsExclusion.findall(cleanTag) and len(re.findall(r'\d', cleanTag)) >= 3:
+                        returnResults.append([{'Phone Number': cleanTag,
+                                               'Entity Type': 'Phone Number'},
+                                              {currentUID: {'Resolution': 'Phone Number Found',
+                                                            'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36',
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractTels(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/PhraseSimilarity.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+class PhraseSimilarity:
+    name = "String Similarity Check"
+    category = "String Operations"
+    description = "Find the ratio of similarity between two strings."
+    originTypes = {'*'}
+    resultTypes = {'Phrase'}
+    parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
+                                             'type': 'SingleChoice',
+                                             'value': {'Notes', 'Primary Field'}},
+                  'Algorithm': {'description': 'Select the Algorithm to use',
+                                'type': 'SingleChoice',
+                                'value': {'levenshtein distance', 'damerau levenshtein distance',
+                                          'jaro distance', 'jaro winkler similarity', 'hamming distance'}}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import jellyfish
+        from itertools import combinations
+
+        return_result = []
+        entity_fields = []
+        selection = parameters['Primary field or Notes']
+        algorithm = parameters['Algorithm'].replace(" ", "_")
+        for entity in entityJsonList:
+            if selection == 'Primary Field':
+                entity_fields.append((entity['uid'], entity[list(entity)[1]].strip()))
+            elif selection == 'Notes':
+                entity_fields.append((entity['uid'], entity.get('Notes')))
+
+        if len(entity_fields) < 2:
+            return "Please Select 2 or more entities for comparison"
+        for combination in combinations(entity_fields, 2):
+            stringA = combination[0][1]
+            uidA = combination[0][0]
+            stringB = combination[1][1]
+            uidB = combination[1][0]
+            similarity = getattr(jellyfish, algorithm)(stringA, stringB)
+            return_result.append([{'Phrase': f'{algorithm} between "{stringA}" and "{stringB}": {similarity}',
+                                   'Entity Type': 'Phrase'},
+                                  {uidA: {'Resolution': f'String Similarity for "{stringA}" and "{stringB}"',
+                                          'Notes': ''},
+                                   uidB: {'Resolution': f'String Similarity for "{stringA}" and "{stringB}"',
+                                          'Notes': ''}}])
+
+        return return_result

Core/Resolutions/Core/RegexMatch.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+
+class RegexMatch:
+    # A string that is treated as the name of this resolution.
+    name = "Get Regex Match"
+
+    category = "String Operations"
+
+    # A string that describes this resolution.
+    description = "Extract text from Phrases and Websites with Regular Expressions."
+
+    originTypes = {'Phrase', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Regex Match': {'description': "Please enter the Regex expression to extract strings with.",
+                                  'type': 'String',
+                                  'value': ''},
+                  'Max Results': {'description': 'Please enter the Maximum number of Results to return.',
+                                  'type': 'String',
+                                  'value': '',
+                                  'default': '5'},
+                  'Re Flags': {'description': 'Select the Regex Flags to be used.',
+                               'type': 'MultiChoice',
+                               'value': {'re.I', 're.M', 're.S'}
+                               }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import requests
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
+        }
+
+        returnResults = []
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if maxResults <= 0:
+            return []
+        search_param = parameters['Regex Match']
+        flags = parameters['Re Flags']
+
+        flagsToUse = 0
+        for flag in flags:
+            flagsToUse |= flag
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if entity['Entity Type'] == 'Phrase':
+                text = str(entity['Notes']) + "\n" + str(entity['Phrase'])
+            else:  # Website entity
+                r = requests.get(entity["URL"], headers=headers)
+                text = r.text
+
+            search_re = re.findall(search_param, text, flags=flagsToUse)
+
+            returnResults.extend([{'Phrase': f'Regex Match: {regexMatch}',
+                                   'Entity Type': 'Phrase',
+                                   'Notes': ''},
+                                  {uid: {'Resolution': 'Regex String Match', 'Notes': ''}}]
+                                 for regexMatch in search_re[:maxResults])
+        return returnResults

Core/Resolutions/Core/ReplacePhrase.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+
+class ReplacePhrase:
+    name = "Replace String in Phrase"
+    category = "String Operations"
+    description = "Replace a character, or sequence of characters in a Phrase with another character or sequence."
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+
+    parameters = {'Sequence to Remove': {'description': 'Specify the character or sequence of characters to replace.',
+                                         'type': 'String',
+                                         'value': ''
+                                         },
+                  'Sequence to Insert': {'description': 'Specify the character or sequence of characters to replace '
+                                                        'the old character or sequence with. Enter the same character '
+                                                        'or sequence to delete the character or sequence instead.',
+                                         'type': 'String',
+                                         'value': ''
+                                         },
+                  'Match Type': {'description': 'Specify whether the matching of characters to replace '
+                                                'should be plain (as in, match characters as they were typed), '
+                                                'case insensitive, or regex.',
+                                 'type': 'SingleChoice',
+                                 'value': {'Plain', 'Case Insensitive', 'Regex'},
+                                 'default': 'Plain'
+                                 },
+                  'Match Count': {'description': 'Specify the number of times to replace the specified character or '
+                                                 'sequence with the new sequence. Zero is unlimited times.',
+                                  'type': 'String',
+                                  'value': '0',
+                                  'default': '0'
+                                  }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+
+        returnResults = []
+
+        remove = parameters['Sequence to Remove']
+        insert = parameters['Sequence to Insert']
+        if insert == remove:
+            insert = ''
+        matchType = parameters['Match Type']
+
+        try:
+            matchCount = int(parameters['Match Count'])
+            if matchCount < 0:
+                return []
+        except ValueError:
+            return "Invalid Match Count specified."
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase']
+
+            if matchType == 'Plain':
+                if matchCount == 0:
+                    matchCount = -1
+                primaryField = primaryField.replace(remove, insert, matchCount)
+            elif matchType == 'Case Insensitive':
+                pattern = re.compile(re.escape(remove), re.IGNORECASE)
+                primaryField = pattern.sub(insert, primaryField, matchCount)
+            else:
+                pattern = re.compile(remove)
+                primaryField = pattern.sub(insert, primaryField, matchCount)
+
+            returnResults.append([{'Phrase': primaryField,
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'Replace characters',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/TikTokVideoPublishDetails.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+class TikTokVideoPublishDetails:
+    # A string that is treated as the name of this resolution.
+    name = "TikTok Video Publishing Details"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Extracts information from public TikTok Videos."
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Social Media Handle', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from datetime import datetime
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            url = entity['URL'].lower()
+
+            try:
+                splitURL = url.split('/')
+                username = splitURL[3]
+                videoID = int(splitURL[5].split('?')[0])
+            except (IndexError, ValueError):
+                continue
+
+            binString = "{0:b}".format(videoID)
+            if len(binString) == 63:
+                binString = f'0{binString}'
+            binString = int(binString[:32], 2)
+
+            UTCTimestamp = f'{datetime.utcfromtimestamp(binString).isoformat()}+00:00'
+
+            returnResults.extend(([{'Date': UTCTimestamp, 'Entity Type': 'Date'},
+                                   {uid: {'Resolution': 'Video Publish Date', 'Notes': ''}}],
+                                  [{'User Name': username, 'Entity Type': 'Social Media Handle'},
+                                   {uid: {'Resolution': 'Published By', 'Notes': ''}}]))
+
+        return returnResults

Core/Resolutions/Core/ToPhrase.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+
+class ToPhrase:
+    name = "Convert To Phrase"
+    category = "String Operations"
+    description = "Convert the primary field of selected entities to a Phrase entity."
+    originTypes = {'*'}
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            returnResults.append([{'Phrase': primaryField,
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'To Phrase',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/WebsiteFromPhrase.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+
+class WebsiteFromPhrase:
+    name = "Website From Phrase"
+    category = "String Operations"
+    description = "Extract website-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        import re
+        import tldextract
+
+        websiteRegex = re.compile(r"""^https?://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$""")
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase']
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if websiteRegex.match(entityChunk):
+                    with contextlib.suppress(Exception):
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'URL': entityChunk,
+                                                   'Entity Type': 'Website'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Website',
+                                                                   'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/WordCounter.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+
+class WordCounter:
+    name = "Get Number Of Words"
+    category = "String Operations"
+    description = "Count the number of words in a given phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
+                                             'type': 'SingleChoice',
+                                             'value': {'Notes', 'Primary Field'}}}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+
+        return_result = []
+        selection = parameters['Primary field or Notes']
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if selection == "Notes":
+                total = len(re.findall(r'\w+', entity['Notes'].strip()))
+            else:
+                total = len(re.findall(r'\w+', entity["Phrase"].strip()))
+            return_result.append([{'Phrase': f"{total} words",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Notes Word Count', 'Notes': ''}}])
+
+        return return_result

Core/ResourceHandler.py

@@ -0,0 +1,538 @@
+#!/usr/bin/env python3
+
+
+import contextlib
+import re
+from typing import Union
+from glob import glob
+
+import networkx as nx
+from datetime import timezone
+from defusedxml.ElementTree import parse
+from datetime import datetime
+from os import listdir
+from pathlib import Path
+from uuid import uuid4
+from ast import literal_eval
+from base64 import b64decode
+from dateutil import parser
+
+from PySide6.QtCore import QByteArray, QSize, QUrl, Qt
+from PySide6 import QtWidgets, QtGui
+
+
+class ResourceHandler:
+
+    def getIcon(self, iconName: str):
+        return self.icons[iconName]
+
+    # Load all resources needed.
+    def __init__(self, mainWindow, messageHandler) -> None:
+        self.mainWindow = mainWindow
+        self.messageHandler = messageHandler
+        self.programBaseDirPath = Path(self.mainWindow.SETTINGS.value("Program/BaseDir"))
+        self.entityCategoryList = {}
+
+        self.icons = {"uploading": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploading.png"),
+                      "uploaded": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploaded.png"),
+                      "upArrow": str(self.programBaseDirPath / "Resources" / "Icons" / "UpArrow.png"),
+                      "downArrow": str(self.programBaseDirPath / "Resources" / "Icons" / "DownArrow.png"),
+                      "isolatedNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectIsolated.png"),
+                      "addCanvas": str(self.programBaseDirPath / "Resources" / "Icons" / "Add_Canvas.png"),
+                      "generateReport": str(self.programBaseDirPath / "Resources" / "Icons" / "Generate_Report.png"),
+                      "leafNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectLeaf.png"),
+                      "nonIsolatedNodes": str(self.programBaseDirPath / "Resources" / "Icons" /
+                                              "SelectNonIsolated.png"),
+                      "rootNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectRoot.png"),
+                      "split": str(self.programBaseDirPath / "Resources" / "Icons" / "Split.png"),
+                      "merge": str(self.programBaseDirPath / "Resources" / "Icons" / "Merge.png"),
+                      "shortestPath": str(self.programBaseDirPath / "Resources" / "Icons" / "ShortestPath.png"),
+                      "drawLink": str(self.programBaseDirPath / "Resources" / "Icons" / "DrawLink.png"),
+                      "rearrange": str(self.programBaseDirPath / "Resources" / "Icons" / "RearrangeGraph.png"),
+                      "colorPicker": str(self.programBaseDirPath / "Resources" / "Icons" / "ColorPicker.png"),
+                      }
+
+        self.banners = {f"{bannerPath.split('Banner_')[-1].split('.')[0]}": str(bannerPath)
+                        for bannerPath in glob(str(self.programBaseDirPath / "Resources" / "Icons" / "Banner_*.svg"))}
+        # These are not meant to be strict - just restrictive enough such that users don't put in utter nonsense.
+        # Note that regex isn't always the best way of validating fields, but it should be good enough for our
+        #   purposes.
+        self.checks = {'Email': re.compile(
+            r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)])"""),
+            'Phonenumber': re.compile(r"""^(\+|00)?[0-9() \-]{3,32}$"""),
+            'String': re.compile(r""".+"""),
+            'URL': re.compile(r"""^(?:(?:http|ftp)s?|file)://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$"""),
+            'Onion': re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$"""),
+            'Domain': re.compile(r"""^(\S(?<!\.)(?!/)(?<!/)){1,63}(\.(\S(?<!\.)(?!/)(?<!/)){1,63})+$"""),
+            'Float': re.compile(r"""^([-+])?(\d|\.(?=\d))+$"""),
+            'WordString': re.compile(r"""^\D+$"""),
+            'Numbers': re.compile(r"""^\d+$"""),
+            'IPv4': re.compile(r"""^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)(\.(?!$)|$)){4}$"""),
+            'IPv6': re.compile(
+                r"""^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9]))$"""),
+            'MAC': re.compile(r"""^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"""),
+            'ASN': re.compile(r"""^(AS)?\d+$"""),
+            'CUSIP': re.compile(r"""^[a-zA-Z0-9]{9}$"""),
+            'EIN': re.compile(r"""^\d{2}-?\d{7}$"""),
+            'LEIID': re.compile(r"""^[a-zA-Z0-9]{20}$"""),
+            'ISINID': re.compile(r"""^[a-zA-Z0-9]{2}-?[a-zA-Z0-9]{9}-?[a-zA-Z0-9]$"""),
+            'SIC/NAICS': re.compile(r"""^[0-9]{4,6}$""")}
+
+        self.loadCoreEntities()
+
+    def getEntityCategories(self) -> list:
+        return list(self.entityCategoryList)
+
+    def getAllEntityDetailsWithIconsInCategory(self, category) -> list:
+        eList = []
+        for entity in self.entityCategoryList[category]:
+            entityValue = self.entityCategoryList[category][entity]
+            eList.append((self.getBareBonesEntityJson(entity),
+                          entityValue['Icon']
+                          ))
+        return eList
+
+    def getEntityAttributes(self, entityType) -> Union[None, list]:
+        aList = []
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    aList.extend(iter(self.entityCategoryList[category][entityType]['Attributes']))
+                    break
+        except KeyError:
+            self.messageHandler.error("Attempted to get attributes for "
+                                      "nonexistent entity type: " + str(entityType), True)
+            return None
+        return aList
+
+    def getAllEntitiesInCategory(self, category) -> list:
+        """
+        Get all Entity Types in the specified category.
+        """
+        return list(self.entityCategoryList[category])
+
+    def getCategoryOfEntityType(self, entityType: Union[str, None]):
+        return next((category for category in self.entityCategoryList
+                     if entityType in self.entityCategoryList[category]),
+                    None)
+
+    def getAllEntities(self) -> list:
+        """
+        Get all recognised Entity Types.
+        """
+        eList = []
+        for category in self.getEntityCategories():
+            eList.extend(iter(self.getAllEntitiesInCategory(category)))
+        return eList
+
+    def validateAttributesOfEntity(self, entityJSON: dict) -> (bool, str):
+        try:
+            entityType = entityJSON['Entity Type']
+            entityCategory = self.getCategoryOfEntityType(entityType)
+            # Attributes that become part of the entity after merging are not checked.
+            # This is fine, because resolutions (by default) don't assume that any extra fields will be present.
+            entityBaseAttributes = self.getEntityAttributes(entityType)
+            if entityCategory is not None:
+                for attribute in self.entityCategoryList[entityCategory][entityType]['Attributes']:
+                    if attribute in entityBaseAttributes:
+                        attrValue = entityJSON.get(attribute)
+                        if attrValue is None or not self.runCheckOnAttribute(
+                                attrValue,
+                                self.entityCategoryList[entityCategory][entityType]['Attributes'][attribute][1]):
+                            return f'Bad value: {str(attrValue)}'
+        except Exception:
+            return False
+        return True
+
+    def runCheckOnAttribute(self, attribute: str, check: str) -> bool:
+        """
+        Check that the attribute value given matches the regex of the category 'check'.
+        """
+        # Ignore checks - this is used for non-string attributes in special entities.
+        if check == 'None':
+            return True
+        attrCheck = self.checks.get(check)
+        if attrCheck is None:
+            return False
+        result = attrCheck.findall(attribute)
+        return len(result) == 1
+
+    def addRecognisedEntityTypes(self, entityFile: Path) -> bool:
+        try:
+            tree = parse(entityFile, forbid_dtd=True, forbid_entities=True, forbid_external=True)
+        except Exception as exc:
+            self.mainWindow.MESSAGEHANDLER.warning('Error occurred when loading entities from '
+                                                   + str(entityFile) + ': ' + str(exc) + ', skipping.')
+            return False
+
+        root = tree.getroot()
+
+        category = root.tag.replace('_', ' ')
+        for entity in list(root):
+            try:
+                entityName = entity.tag.replace('_', ' ')
+                attributes = entity.find('Attributes')
+                primaryCount = 0
+                attributesDict = {}
+                for attribute in list(attributes):
+                    defaultValue = attribute.attrib['default']
+                    valueCheck = attribute.attrib['check']
+                    isPrimary = attribute.attrib['primary'] == 'True'
+                    if isPrimary:
+                        if primaryCount > 0:
+                            raise AttributeError(f'Malformed Entity: {entityName} - too many primary fields')
+                        else:
+                            primaryCount += 1
+                    if not self.runCheckOnAttribute(defaultValue, valueCheck):
+                        raise AttributeError(f'Malformed Entity: {entityName} - default values do not pass their '
+                                             f'corresponding checks.')
+                    attributeName = attribute.text
+                    attributesDict[attributeName] = [attribute.attrib['default'], attribute.attrib['check'], isPrimary]
+                if primaryCount != 1:
+                    raise AttributeError(f'Malformed Entity: {entityName} - invalid number of primary fields '
+                                         f'specified.')
+
+                icon = entity.find('Icon')
+                icon = icon.text.strip() if icon is not None else 'Default.svg'
+                if self.entityCategoryList.get(category) is None:
+                    self.entityCategoryList[category] = {}
+                self.entityCategoryList[category][entityName] = {
+                    'Attributes': attributesDict,
+                    'Icon': str(self.programBaseDirPath / "Resources" / "Icons" / icon)}
+            except (KeyError, AttributeError) as err:
+                # Ignore malformed entities
+                self.messageHandler.error(f'Error: {str(err)}', popUp=False)
+                continue
+        return True
+
+    def loadCoreEntities(self) -> None:
+        entDir = self.programBaseDirPath / "Core" / "Entities"
+        for entFile in listdir(entDir):
+            if entFile.endswith('.xml'):
+                self.addRecognisedEntityTypes(entDir / entFile)
+
+    def loadModuleEntities(self) -> None:
+        entDir = self.programBaseDirPath / "Modules"
+        for module in listdir(entDir):
+            for entFile in listdir(entDir / module):
+                if entFile.endswith('.xml'):
+                    self.addRecognisedEntityTypes(
+                        entDir / module / entFile)
+
+    def getEntityJson(self, entityType: str, jsonData=None) -> Union[dict, None]:
+        eJson = {'uid': str(uuid4())}
+        if entityType in self.getAllEntitiesInCategory('Meta'):
+            eJson['uid'] += '@'
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
+                    break
+        except KeyError:
+            self.messageHandler.error(f"Attempted to get attributes for malformed entity type: {entityType}", True)
+            return None
+        eJson['Entity Type'] = entityType
+        eJson['Date Created'] = None
+        eJson['Date Last Edited'] = None
+        eJson['Notes'] = ""
+        eJson['Icon'] = self.getEntityDefaultPicture(entityType)
+
+        if jsonData is not None:
+            # Allow setting of attributes that are not defined in the Entity specification.
+            for key in jsonData:
+                value = jsonData.get(key)
+                if value is not None and value != '':
+                    eJson[key] = value
+
+        utcNow = datetime.isoformat(datetime.now(timezone.utc))
+        if eJson['Date Created'] is None:
+            eJson['Date Created'] = utcNow
+        else:
+            # Always make sure dates are in ISO format.
+            try:
+                eJson['Date Created'] = parser.parse(str(eJson['Date Created'])).isoformat()
+            except (TypeError, ValueError):
+                eJson['Date Created'] = utcNow
+
+        eJson['Date Last Edited'] = utcNow
+
+        return eJson
+
+    def getPrimaryFieldForEntityType(self, entityType: str) -> Union[str, None]:
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        if self.entityCategoryList[category][entityType]['Attributes'][attribute][2]:
+                            return attribute
+        except KeyError:
+            self.messageHandler.error(f"Attempted to get primary attribute for malformed entity type: {entityType}",
+                                      True)
+        return None
+
+    def getBareBonesEntityJson(self, entityType: str) -> Union[dict, None]:
+        eJson = {}
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
+                    break
+        except KeyError:
+            self.messageHandler.error(f"Attempted to get attributes for malformed entity type: {entityType}", True)
+            return None
+        eJson['Entity Type'] = entityType
+
+        return eJson
+
+    def getLinkJson(self, jsonData: dict) -> Union[dict, None]:
+        linkJson = {}
+        try:
+            linkJson['uid'] = jsonData['uid']
+        except KeyError:
+            return None
+
+        utcNow = datetime.isoformat(datetime.now(timezone.utc))
+        linkJson['Resolution'] = str(jsonData.get('Resolution'))  # This way, if it is None, it is cast to a string.
+        linkJson['Date Created'] = jsonData.get('Date Created')
+        # Make sure that dates are always in ISO format.
+        if linkJson['Date Created'] is None:
+            linkJson['Date Created'] = utcNow
+        else:
+            try:
+                linkJson['Date Created'] = parser.parse(str(linkJson['Date Created'])).isoformat()
+            except (TypeError, ValueError):
+                linkJson['Date Created'] = utcNow
+        linkJson['Date Last Edited'] = utcNow
+        linkJson['Notes'] = str(jsonData.get('Notes', ""))
+
+        # Transfer all values from jsonData to linkJson, but preserve the values and order of linkJson for existing
+        #   keys.
+        jsonData |= linkJson
+        linkJson |= jsonData
+
+        return linkJson
+
+    def getEntityDefaultPicture(self, entityType) -> QByteArray:
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Default.svg"
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    entityPicture = self.entityCategoryList[category][entityType]['Icon']
+                    if Path(entityPicture).exists():
+                        picture = entityPicture
+                    break
+        except KeyError:
+            self.messageHandler.warning("Attempted to get icon for "
+                                        "nonexistent entity type: " + str(entityType), popUp=False)
+        finally:
+            with open(picture, 'rb') as pictureFile:
+                pictureContents = pictureFile.read()
+            return QByteArray(pictureContents)
+
+    def getLinkPicture(self):
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Resolution.png"
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
+
+    def getLinkArrowPicture(self):
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Right-Arrow.svg"
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
+
+    def deconstructGraph(self, graph: nx.DiGraph) -> tuple:
+        nodes = {}
+        for nodeKey in graph.nodes:
+            # Dereference the original dict, so we don't actually convert its icon to data.
+            nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
+            with contextlib.suppress(KeyError):
+                nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
+        edges = {edgeKey: graph.edges.get(edgeKey) for edgeKey in graph.edges}
+        return nodes, edges
+
+    def deconstructGraphForFileDump(self, graph: nx.DiGraph) -> tuple:
+        nodes = {}
+        for nodeKey in graph.nodes:
+            # Dereference the original dict, so we don't actually convert its icon to data.
+            nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
+            with contextlib.suppress(KeyError):
+                nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
+        edges = {str(edgeKey): graph.edges.get(edgeKey) for edgeKey in graph.edges}
+        return nodes, edges
+
+    def reconstructGraphFromString(self, graphString: str) -> tuple:
+        nodes, edges = literal_eval(graphString)
+        for node in nodes:
+            with contextlib.suppress(KeyError):
+                nodes[node]['Icon'] = QByteArray(b64decode(nodes[node]['Icon']))
+        return nodes, edges
+
+    def reconstructGraphFullFromFile(self, graphNodesAndEdges: Union[tuple, list]) -> nx.DiGraph:
+        returnGraph = nx.DiGraph()
+        graphNodes = graphNodesAndEdges[0]
+        graphEdges = graphNodesAndEdges[1]
+        for node in graphNodes:
+            with contextlib.suppress(KeyError):
+                graphNodes[node]['Icon'] = QByteArray(b64decode(graphNodes[node]['Icon']))
+            returnGraph.add_node(node, **graphNodes[node])
+
+        for edge in graphEdges:
+            edgeUID = tuple(literal_eval(edge))
+            graphEdges[edge]['uid'] = edgeUID
+            returnGraph.add_edge(*edgeUID, **graphEdges[edge])
+
+        return returnGraph
+
+
+class StringPropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(StringPropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+
+    def getValue(self):
+        return self.text()
+
+
+class FilePropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(FilePropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+        self.fileDialog = QtWidgets.QFileDialog()
+
+    def getValue(self):
+        return self.text()
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        fileChosen = self.fileDialog.getOpenFileName(self,
+                                                     "Open File",
+                                                     str(Path.home()),
+                                                     options=QtWidgets.QFileDialog.Option.DontUseNativeDialog)
+        self.setText(fileChosen[0])
+
+
+class SingleChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOption):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(SingleChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOption is None:
+            defaultOption = ''
+
+        for option in enforceOptionsSet:
+            radioButton = QtWidgets.QRadioButton(option)
+            if option == defaultOption:
+                radioButton.setChecked(True)
+            else:
+                radioButton.setChecked(False)
+            self.options.append(radioButton)
+            vboxLayout.addWidget(radioButton)
+
+    def getValue(self):
+        return next((option.text() for option in self.options if option.isChecked()), '')
+
+
+class MultiChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOptions):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(MultiChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOptions is None:
+            defaultOptions = []
+
+        for option in enforceOptionsSet:
+            checkBox = QtWidgets.QCheckBox(option)
+            if option in defaultOptions:
+                checkBox.setChecked(True)
+            else:
+                checkBox.setChecked(False)
+            self.options.append(checkBox)
+            vboxLayout.addWidget(checkBox)
+
+    def getValue(self):
+        return [option.text() for option in self.options if option.isChecked()]
+
+
+class MinSizeStackedLayout(QtWidgets.QStackedLayout):
+    """
+    Resize the layout to always take up the appropriate space for the currently selected widget.
+    Otherwise, large widgets (due to selecting entities with long strings of text) will stretch
+    out the ScrollArea and make the other, non-selected widgets to look bad when the layout
+    switches over.
+
+    https://stackoverflow.com/a/34300567
+    """
+
+    def sizeHint(self) -> QSize:
+        return self.currentWidget().sizeHint()
+
+    def minimumSize(self) -> QSize:
+        return self.currentWidget().minimumSize()
+
+
+class RichNotesEditor(QtWidgets.QTextBrowser):
+
+    def __init__(self, parent=None, currentText: str = '#### Type notes here.\n', allowEditing: bool = True):
+        super(RichNotesEditor, self).__init__(parent=parent)
+        self.allowEditing = allowEditing
+        self.setReadOnly(True)
+        if allowEditing:
+            self.setUndoRedoEnabled(True)
+            self.setTextInteractionFlags(Qt.TextBrowserInteraction | Qt.TextSelectableByKeyboard)
+        else:
+            self.setUndoRedoEnabled(False)
+            self.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.TextSelectableByKeyboard
+                                         | Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard)
+
+        self.contents = currentText
+        self.setMarkdown(self.contents)
+        self.textFormat = self.currentCharFormat()
+
+    def startEditing(self) -> None:
+        if self.allowEditing and self.isReadOnly():
+            # Reset char format to plain text.
+            self.setCurrentCharFormat(self.textFormat)
+            self.setPlainText(self.contents)
+            self.setReadOnly(False)
+
+    def stopEditing(self) -> None:
+        if not self.isReadOnly():
+            self.contents = self.toPlainText()
+            self.setMarkdown(self.contents)
+            self.setReadOnly(True)
+
+    def dropEvent(self, e: QtGui.QDropEvent) -> None:
+        if self.allowEditing:
+            editingBefore = self.isReadOnly()
+            self.startEditing()
+            super(RichNotesEditor, self).dropEvent(e)
+            if editingBefore:
+                self.stopEditing()
+
+    def mousePressEvent(self, ev: QtGui.QMouseEvent) -> None:
+        potentialLink = self.anchorAt(ev.pos())
+        if not potentialLink and ev.button() == QtGui.Qt.MouseButton.LeftButton:
+            self.startEditing()
+        super(RichNotesEditor, self).mousePressEvent(ev)
+
+    def focusOutEvent(self, ev: QtGui.QFocusEvent) -> None:
+        if not self.underMouse() and self.isActiveWindow():
+            self.stopEditing()
+        super(RichNotesEditor, self).focusOutEvent(ev)
+
+    def doSetSource(self, name: Union[QUrl, str], resourceType: QtGui.QTextDocument.ResourceType = ...) -> None:
+        QtGui.QDesktopServices.openUrl(name)

Core/SettingsObject.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+from msgpack import dump
+from shutil import move
+from pathlib import Path
+from Core.PathHelper import is_path_exists_or_creatable_portable
+from PySide6.QtCore import QSettings
+
+
+class SettingsObject(dict):
+    """
+    Stores the following session settings:
+
+    Project/Name: Self explanatory.
+    Project/BaseDir: The directory in which the current project is stored.
+    Program/BaseDir: The directory in which the program is stored.
+    MainWindow/Geometry: Stores the window size
+    MainWindow/WindowState: Saves window state (toolbar position etc.)
+    Logging/Severity: Dictates logging severity
+    Logging/Logfile: Which file should the software write logs to.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.globalSettings = QSettings()
+        self.globalSettings.setValue("Program/Version", "v1.5.0")
+        self.globalSettings.setValue("Program/TOR Profile Location",
+                                     self.globalSettings.value("Program/TOR Profile Location", ""))
+        self.globalSettings.setValue("Program/BaseDir",
+                                     self.globalSettings.value("Program/BaseDir", "Unset"))
+
+        # The value '20' equates to logging.INFO
+        # It's not necessary to set this, but we will for
+        #   the sake of completeness
+        self.globalSettings.setValue("Logging/Severity",
+                                     self.globalSettings.value("Logging/Severity", "20"))
+        self.globalSettings.setValue("Logging/Logfile",
+                                     self.globalSettings.value("Logging/Logfile",
+                                                               str(Path.home() / 'LinkScope_logfile.log')))
+
+        self.globalSettings.setValue("Program/Graph Layout",
+                                     self.globalSettings.value("Program/Graph Layout", "dot"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Type",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Type", "Mono"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Size",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Size", "11"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Boldness",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Boldness", "700"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Type",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Type", "Mono"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Size",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Size", "11"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Boldness",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Boldness", "700"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Color",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Color", "#000000"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Color",
+                                     self.globalSettings.value("Program/Graphics/Link Text Color", "#000000"))
+        self.globalSettings.setValue("Program/Graphics/Label Fade Scroll Distance",
+                                     self.globalSettings.value("Program/Graphics/Label Fade Scroll Distance", "3"))
+
+        self.setValue("Project/Name", "Untitled")
+        self.setValue("Project/BaseDir", "")
+        self.setValue("Project/FilesDir", "")
+        # For any entity with a Path variable, this dictates whether a copy of the original is made or whether a
+        #   symlink is created. Symlinks however require special permissions or developer mode in Windows.
+        # To ensure that the software works out-of-the-box on all platforms, the default is set to 'Copy'.
+        self.setValue("Project/Symlink or Copy Materials", "Copy")  # Values are 'Copy' or 'Symlink'.
+        self.setValue("Project/Resolution Result Grouping Threshold", "15")
+        self.setValue("Project/Number of Answers Returned", "3")
+        self.setValue("Project/Question Answering Retriever Value", "10")
+        self.setValue("Project/Question Answering Reader Value", "10")
+        self.setValue("Project/Server/Project", "")
+        self.setValue("Project/Server/Collectors", "{}")
+
+    def getGroupSettings(self, settingsGroup: str) -> dict:
+        if not settingsGroup.endswith('/'):
+            settingsGroup += '/'
+        settingsDict = {}
+        for setting in self.globalSettings.allKeys():
+            if setting.startswith(settingsGroup):
+                settingsDict[setting] = self.globalSettings.value(setting)
+        for setting in self:
+            if setting.startswith(settingsGroup):
+                settingsDict[setting] = self[setting]
+        return dict(sorted(settingsDict.items()))
+
+    def setValue(self, key, value) -> None:
+        if self.globalSettings.contains(key):
+            self.globalSettings.setValue(key, value)
+        self[key] = value
+
+    def setGlobalValue(self, key, value) -> None:
+        """
+        Helper in the case we want to be explicit in setting a value globally.
+        """
+        self.globalSettings.setValue(key, value)
+
+    def value(self, key, alt=None):
+        if self.globalSettings.contains(key):
+            return self.globalSettings.value(key)
+        return self.get(key, alt)
+
+    def removeKey(self, key) -> bool:
+        try:
+            if self.globalSettings.contains(key):
+                self.globalSettings.remove(key)
+            else:
+                self.pop(key)
+            return True
+        except KeyError:
+            return False
+
+    def save(self) -> None:
+        # Save and then move to prevent corruption if the application closes unexpectedly.
+        actualSavePath = str(Path(self.value("Project/BaseDir")).joinpath(self.value("Project/Name") + ".linkscope"))
+        if is_path_exists_or_creatable_portable(actualSavePath):
+            tempSavePath = f'{actualSavePath}.tmp'
+            with open(tempSavePath, "wb") as projectFile:
+                dump(self, projectFile)
+            move(tempSavePath, actualSavePath)
+        self.globalSettings.sync()
+        globalSettingsSavingError = self.globalSettings.status()
+        if globalSettingsSavingError != self.globalSettings.Status.NoError:
+            raise Exception(f'Could not save global settings: {globalSettingsSavingError}')
+
+    def load(self, savedDict: dict) -> None:
+        # No need to do anything with global settings.
+        for key in savedDict:
+            self[key] = savedDict[key]

Core/URLManager.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+import zipfile
+import magic
+from pathlib import Path
+from os import symlink
+from shutil import copy2
+from hashlib import sha3_512
+from binascii import hexlify
+from urllib.parse import urlparse
+
+from PySide6 import QtCore
+
+
+class URLManager:
+    """
+    This class handles URLs. If they are local, the appropriate entity
+    is created to represent the type of file (Document, Image etc).
+
+    The only archive type supported as of now is Zip.
+
+    Note that there is no need to add entities to the database here, as
+    they will be added by the class that is calling functionality from this one.
+    """
+
+    def __init__(self, mainWindow):
+        self.mainWindow = mainWindow
+
+    def handleURLs(self, urls):
+        """
+        Takes a list of QUrls and returns a list of entities that correspond
+        to them.
+        """
+        return [self.handleURL(url) for url in urls]
+
+    def handleURL(self, url):
+        parsedURL = urlparse(url.toString())
+        if not url.isValid() or (not all([parsedURL.scheme, parsedURL.netloc]) and parsedURL.scheme != 'file'):
+            return None
+        if url.isLocalFile():
+            return self.handleLocalURL(url)
+        else:
+            return self.handleRemoteURL(url)
+
+    def handleURLString(self, urlString):
+        return self.handleURL(QtCore.QUrl(urlString))
+
+    def handleLocalURL(self, url):
+        urlPath = Path(url.toLocalFile())
+
+        urlName = urlPath.name
+        urlPathString = str(urlPath)
+        savePathString = str(self.moveURLToProjectFilesHelperIfNeeded(urlPath))
+
+        if savePathString == 'None':
+            return None
+
+        fileType = magic.from_file(urlPathString, mime=True)
+        fileTypeSplit1, fileTypeSplit2 = fileType.split('/', 1)
+        # CSV files not considered - may have any dialect, hard to accommodate.
+        if urlPath.suffix in ('.ods', '.xls', '.xlsm', '.xlsx') and \
+                fileTypeSplit2 in ('vnd.oasis.opendocument.spreadsheet',
+                                   'vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+                                   'vnd.ms-excel',
+                                   'vnd.openxmlformats-officedocument.spreadsheetml.sheet'):
+            entityJson = {"Spreadsheet Name": urlName,
+                          "File Path": savePathString,
+                          "Entity Type": "Spreadsheet"}
+        # Only support zip files for archives (for now) 10/Jul/2021).
+        elif zipfile.is_zipfile(urlPathString):
+            entityJson = {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
+        elif fileTypeSplit1 == "video":
+            entityJson = {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
+        elif fileTypeSplit1 == "image":
+            entityJson = {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
+        else:
+            entityJson = {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
+        return entityJson
+
+    def moveURLToProjectFilesHelperIfNeeded(self, urlPath: Path):
+        valuePath = Path(urlPath).absolute()
+
+        if not valuePath.exists() or not valuePath.is_file():
+            return None
+
+        projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).absolute()
+        try:
+            savePath = valuePath.relative_to(projectFilesPath)
+        except ValueError:
+            # The file selected is not in Project Files
+            createSymlink = self.mainWindow.SETTINGS.value("Project/Symlink or Copy Materials") == "Symlink"
+
+            projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir"))
+            # Create a unique path in Project Files
+            saveHash = hexlify(sha3_512(str(urlPath).encode()).digest()).decode()[:16]  # nosec
+            savePath = projectFilesPath / f'{saveHash}|{urlPath.name}'
+
+            if createSymlink:
+                symlink(urlPath, savePath)
+            else:
+                copy2(urlPath, savePath)
+
+            savePath = savePath.relative_to(projectFilesPath)
+
+        return savePath
+
+    def handleRemoteURL(self, url):
+        stringURL = url.toString()
+        return {'Entity Type': 'Onion Website', 'Onion URL': stringURL} \
+            if self.mainWindow.RESOURCEHANDLER.runCheckOnAttribute(stringURL, 'Onion') \
+            else {'Entity Type': 'Website', 'URL': stringURL}

Installer/LinuxInstaller.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Spec for .desktop files:
+# https://specifications.freedesktop.org/menu-spec/latest/apcs02.html
+# https://specifications.freedesktop.org/menu-spec/latest/index.html
+
+read -r -d '' HELPTEXT << EOF
+== Installation script for LinkScope Client for Ubuntu 20.04 or greater ==
+
+Usage:
+------
+bash $0 install
+- or -
+bash $0 uninstall
+
+Instructions:
+-------------
+Run the script as a user with sudo permissions, with either 'install' or 'uninstall' (no quotes) as the one and only parameter.
+The script will then install or uninstall the software for all users on the computer.
+
+Function:
+---------
+The script will update the package repositories, and attempt to install the following packages:
+p7zip-full curl libopengl0 graphviz libmagic1
+
+Then, the script will download the latest version of the software from AccentuSoft's repository, add it to /usr/local/sbin, and optionally create a Desktop shortcut for the software.
+EOF
+
+read -r -d '' DESKTOP_ENTRY << EOF
+[Desktop Entry]
+Name=LinkScope Client
+StartupWMClass=LinkScope Client
+Comment=Start LinkScope Client
+GenericName=Investigation Software
+Terminal=false
+Exec=/usr/local/sbin/LinkScope/LinkScope
+Icon=/usr/local/sbin/LinkScope/Icon.ico
+Type=Application
+Categories=Application;Office;DataVisualization;
+MimeType=application/linkscope
+Keywords=LinkScope;Investigation;Graph;Knowledge
+EOF
+
+
+# Make sure we're not running as root.
+if [ "$EUID" -eq 0 ]; then
+  echo "Please run the installation script as a user with sudo permissions, not as root."
+  exit
+fi
+
+if [ "$#" -ne 1 ]; then
+  echo "$HELPTEXT"
+  exit
+fi
+
+echo "Sudo permission check"
+sudo echo "Sudo permissions available"
+
+if [ "$?" -ne 0 ]; then
+  echo "Sudo permissions not available, aborting."
+  exit
+fi
+
+if [ "$1" == "install" ]; then
+  echo "Installing Software"
+  echo "Updating Packages"
+  sudo apt update
+  echo "Installing New Packages"
+  sudo apt install p7zip-full curl libopengl0 graphviz libmagic1 -y
+  echo "Downloading latest version of LinkScope client..."
+  linuxURL=$(curl -sL https://github.com/AccentuSoft/LinkScope_Client/releases/latest | grep 'Ubuntu-x64.7z' -m 1 | cut -d '"' -f 2 | tr -d ' ')
+  curl -L https://github.com${linuxURL} -o /tmp/LinkScope.7z
+  sudo 7z x /tmp/LinkScope.7z -o/usr/local/sbin/ && rm /tmp/LinkScope.7z
+  if [ $? -ne 0 ]; then
+    echo "Something went wrong during the download or extraction."
+    echo "Please check that /tmp/LinkScope.7z exists, and that it is an archive containing the latest version of the LinkScope Client software."
+    exit
+  fi
+  sudo echo "$DESKTOP_ENTRY" > /tmp/LinkScope.desktop
+  sudo mv /tmp/LinkScope.desktop /usr/share/applications/LinkScope.desktop
+  sudo chmod +x /usr/share/applications/LinkScope.desktop
+  read -p "Create a Desktop shortcut? WARNING: This will refresh the desktop! [y/N]" -n 1 -r
+  # https://askubuntu.com/a/1014261    -- Making Desktop launchers with .desktop files
+  # https://stackoverflow.com/a/62240478   -- Making .desktop files work.
+  # Not refreshing the desktop would mean that the desktop icon starts working after a restart.
+  # Manually creating a desktop icon through the gui is simpler; after creating a symlink, the user can right-click
+  #   the .desktop file and select 'Allow Launching'. This however requires user interaction.
+  if [[ $REPLY =~ ^[Yy]$ ]]; then
+    ln -s /usr/share/applications/LinkScope.desktop ${HOME}/Desktop/LinkScope.desktop
+    dbus-launch gio set ${HOME}/Desktop/LinkScope.desktop "metadata::trusted" true
+    dbus-send --type=method_call --print-reply --dest=org.gnome.Shell /org/gnome/Shell org.gnome.Shell.Eval string:'global.reexec_self()'
+  fi
+  echo "Done."
+  exit
+fi
+
+if [ "$1" == "uninstall" ]; then
+  echo "Removing Software"
+  sudo rm -rf /usr/local/sbin/LinkScope
+  sudo rm /usr/share/applications/LinkScope.desktop
+  rm ${HOME}/Desktop/LinkScope.desktop
+
+  echo "Done."
+  echo "Note that the system packages used by the software are not removed, in the case that they are necessary for other software to function."
+  exit
+fi
+
+# If the argument passed in is not 'install' or 'uninstall', print the help message.
+echo "$HELPTEXT"

Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByDealer:
+    name = "Aircraft Inquiry By Dealer"
+    category = "Aircraft"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Company"}
+    resultTypes = {'Phrase', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import pandas as pd
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}DealerResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Dealertxt": entity['Company Name']}))
+            for future in as_completed(futures):
+                uid = uidList[futures.index(future)]
+                try:
+                    df_list = pd.read_html(future.result().text)
+                except requests.exceptions.ConnectionError:
+                    return "Please check your internet connection"
+                except ValueError:
+                    return_result = []
+                    return return_result
+                df = df_list[0]
+                for certificate_index in range(len(df["Certificate Number"])):
+                    index_of_child = len(return_result)
+                    return_result.append([{'Company Name': df["Name"][certificate_index],
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'Aircraft Dealer', 'Notes': ''}}])
+                    return_result.append([{'Phrase': df["Certificate Number"][certificate_index],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Aircraft Certificate Number', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByEngine:
+    name = "Aircraft Inquiry By Engine"
+    category = "Aircraft"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Phrase"}
+    resultTypes = {"Phrase"}
+    parameters = {'Manufacturer': {'description': "Enter the Manufacturer of the Engine Model",
+                                   'type': 'String',
+                                   'value': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        Manufacturer = parameters['Manufacturer']
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}EngineReferenceResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Modeltxt": entity['Phrase'],
+                                                               "MfrNametxt": Manufacturer}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            return_result.append([{'Phrase': f"Model Code:str({df['Mfr/Mdl Code']})",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Model Code', 'Notes': ''}}])
+            return_result.append([{'Phrase': f"Engine Type:{df['Type Engine']}",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Engine Type', 'Notes': ''}}])
+            return_result.append([{'Phrase': f"Horse Power:str({df['Horsepower']})",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Engine Horsepower', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByNNumber:
+    name = "Aircraft Inquiry By N-Number"
+    category = "Aircraft"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Phrase"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}NNumberResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"NNumbertxt": entity['Phrase']}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df1 = df_list[0]
+            df2 = df_list[1]
+            df3 = df_list[2]
+            return_result.append([{'ID Number': df1[1][0],
+                                   'Entity Type': 'Identification Number'},
+                                  {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
+            return_result.append([{'Company Name': df1[1][1],
+                                   'Entity Type': 'Company'},
+                                  {uid: {'Resolution': 'Aircraft Company', 'Notes': ''}}])
+            return_result.append([{'Full Name': df2[1][0],
+                                   'Entity Type': 'Person'},
+                                  {uid: {'Resolution': 'Aircraft Owner', 'Notes': ''}}])
+            return_result.append([{'City Name': df2[1][2],
+                                   'Entity Type': 'City'},
+                                  {uid: {'Resolution': "Aircraft Owner's City", 'Notes': ''}}])
+            return_result.append([{'Country Name': df2[1][4],
+                                   'Entity Type': 'Country'},
+                                  {uid: {'Resolution': "Aircraft Owner's Country", 'Notes': ''}}])
+            return_result.append([{'Phrase': df3[1][1],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': "Aircraft Engine Series", 'Notes': ''}}])
+            return_result.append([{'Phrase': df3[1][2],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': "Aircraft Engine Motor", 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByPersonName:
+    name = "Aircraft Inquiry By Person Name"
+    category = "Aircraft"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Person"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}NameResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"nametxt": entity['Full Name'], "sort_option": "1"}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            for i in range(len(df["N-Number"])):
+                return_result.append([{'Phrase': df["N-Number"][0],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
+                return_result.append([{'ID Number': str(df['Serial Number'][0]),
+                                       'Entity Type': 'Identification Number'},
+                                      {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
+                return_result.append([{'Company Name': df['Manufacturer Name Model'][0],
+                                       'Entity Type': 'Company'},
+                                      {uid: {'Resolution': 'Aircraft Manufacturer Name', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryBySerialNumber:
+    name = "Aircraft Inquiry By Serial Number"
+    category = "Aircraft"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Identification Number"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}SerialResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Serialtxt": entity['ID Number'], "sort_option": "1"}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            for i in range(len(df["N-Number"])):
+                return_result.append([{'Company Name': df["Manufacturer Name"][i],
+                                       'Entity Type': 'Company'},
+                                      {uid: {'Resolution': 'Aircraft Manufacturer', 'Notes': ''}}])
+                return_result.append([{'Phrase': df["N-Number"][i],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
+                return_result.append([{'Phrase': df["Model"][i],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft Model', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/requirements.txt

@@ -0,0 +1,3 @@
+requests
+pandas
+requests-futures

Modules/Aleph/Aleph.xml

@@ -0,0 +1,18 @@
+<Aleph>
+  <Aleph_ID>
+    <Attributes>
+      <Attribute default="Aleph Default ID" check="String" primary="True">ID</Attribute>
+    </Attributes>
+    <Icon>
+      Default.svg
+    </Icon>
+  </Aleph_ID>
+  <Aleph_Collection_ID>
+    <Attributes>
+      <Attribute default="Aleph Default Collection ID" check="String" primary="True">ID</Attribute>
+    </Attributes>
+    <Icon>
+      Default.svg
+    </Icon>
+  </Aleph_Collection_ID>
+</Aleph>

Modules/Aleph/Aleph_Entity_Search.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+
+
+class Aleph_Entity_Search:
+    name = "Aleph Entity Search"
+    category = "Aleph OCCRP"
+    description = "Find information about a given search parameter"
+    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
+    resultTypes = {'Phrase'}
+    parameters = {'Max Results': {'description': 'The maximum number of results to return.',
+                                  'type': 'String',
+                                  'value': 'Enter the number of results you want returned',
+                                  'default': '1'},
+                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import pycountry
+        import time
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        return_result = []
+        uidList = []
+        futures = []
+
+        url = "https://aleph.occrp.org/api/2/entities"
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        gender = "None"
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        try:
+            max_results = int(parameters['Max Results'])
+        except ValueError:
+            return "The value for parameter 'Max Results' is not a valid integer."
+
+        if max_results <= 0:
+            return []
+
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                crafted_url = url + f"?q={primary_field}&filter:schemata=Thing&limit={max_results}"
+                time.sleep(1)
+                futures.append(session.get(crafted_url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            # print(response)
+            for schema in response['results']:
+                index_of_child = len(return_result)
+                try:
+                    if schema['schema'] == "Person":
+                        if schema['properties'].get('gender') is not None \
+                                and schema['properties'].get('gender')[0] == "F":
+                            gender = "Female"
+                        elif schema['properties'].get('gender') is not None \
+                                and schema['properties'].get('gender')[0] == "M":
+                            gender = "Male"
+                        if schema['properties'].get('legalForm') is not None:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties'].get('birthDate')),
+                                  'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}",
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Full Name': str(schema['properties']['name'][0]),
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['birthDate'][0]),
+                                  'Notes': schema['links']['self'],
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
+                        if schema['properties'].get('registrationNumber') is not None:
+                            return_result.append(
+                                [{'Registration Number': str(schema['properties']['registrationNumber'][0]),
+                                  'Notes': '',
+                                  'Entity Type': 'Company'},
+                                 {index_of_child: {'Resolution': 'Aleph Registration Number', 'Notes': ''}}])
+                        if schema['properties'].get('country') is not None:
+                            return_result.append(
+                                [{'Country Name': str(
+                                    pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
+                                    'Notes': '',
+                                    'Entity Type': 'Country'},
+                                    {index_of_child: {'Resolution': 'Country of Origin', 'Notes': ''}}])
+                        if schema['properties'].get('addressEntity'):
+                            return_result.append(
+                                [{'Street Address': str(
+                                    schema['properties']['addressEntity'][0]['properties']['full'][0]),
+                                    'Notes': '',
+                                    'Entity Type': 'Address'},
+                                    {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Street Address': str(schema['properties']['address'][0]),
+                                  'Notes': '',
+                                  'Entity Type': 'Address'},
+                                 {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph Collection ID'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                    elif schema['schema'] == "Organization":
+                        return_result.append(
+                            [{'Organization Name': str(schema['properties']['name'][0]),
+                              'Registration Number': str(schema['properties']['registrationNumber'][0]),
+                              'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}\n"
+                                       f"Source URL: {schema['properties']['sourceUrl'][0]}",
+                              'Entity Type': 'Organization'},
+                             {uid: {'Resolution': 'Aleph Organisation Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Country Name': str(
+                                pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
+                                'Notes': '',
+                                'Entity Type': 'Country'},
+                                {index_of_child: {'Resolution': "Aleph Organisation Country", 'Notes': ''}}])
+                        return_result.append(
+                            [{'Street Address': str(schema['properties']['address'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Address'},
+                             {index_of_child: {'Resolution': "Aleph Organisation Address", 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': "Aleph Organisation ID", 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phone Number': str(schema['properties']['phone'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Phone Number'},
+                             {index_of_child: {'Resolution': 'Phone Number', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['classification'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Organisation Classification', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                    elif schema['schema'] == "Pages":
+                        if 'updated_at' in schema:
+                            date_created = schema['updated_at']
+                        else:
+                            date_created = schema['created_at']
+                        doc_name = 'Document: ' + schema['properties']['title'][0]
+                        entity_link = schema['links']['self']
+                        file_link = schema['links']['file']
+                        source_url = schema['properties']['sourceUrl'][0]
+                        return_result.append(
+                            [{'Phrase': doc_name,
+                              'Source': source_url,
+                              'Notes': 'Link to Aleph Entity: ' + entity_link + '\n\n' +
+                                       'Link to document: ' + file_link,
+                              'Entity Type': 'Phrase',
+                              'Date Created': date_created},
+                             {uid: {'Resolution': 'Aleph Document', 'Notes': ''}}])
+                    elif schema['properties']['parent'][0]['schema'] == "Person":
+                        gender = str(schema['properties']['parent'][0]['properties'].get('gender')[0])
+                        if schema['properties']['parent'][0]['properties'].get('legalForm') is not None:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
+                                  'Notes': f"{schema['properties']['parent'][0]['links']['self']}\nLegal Form: "
+                                           f"{schema['properties']['parent'][0]['properties']['legalForm'][0]}",
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
+                                  'Notes': schema['properties']['parent'][0]['links']['self'],
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
+                        if schema['properties']['parent'][0]['properties'].get('registrationNumber') is not None:
+                            return_result.append(
+                                [{'Registration Number': str(
+                                    schema['properties']['parent'][0]['properties']['registrationNumber'][0]),
+                                    'Notes': '',
+                                    'Entity Type': 'Company'},
+                                    {index_of_child: {'Resolution': 'Company Registration Number', 'Notes': ''}}])
+                        if schema['properties']['parent'][0]['properties'].get('country') is not None:
+                            return_result.append(
+                                [{'Country Name': str(
+                                    pycountry.countries.get(
+                                        alpha_2=schema['properties']['parent'][0]['properties']['country'][0]).name),
+                                    'Notes': '',
+                                    'Entity Type': 'Country'},
+                                    {index_of_child: {'Resolution': 'Country', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['properties']['parent'][0]['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['label']),
+                              'Notes': str(schema['properties']['parent'][0]['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                        index_of_child_of_child = len(return_result)
+                        return_result.append(
+                            [{'Company Name': str(schema['properties']['name'][0]),
+                              'Notes': schema['links']['self'],
+                              'Entity Type': 'Company'},
+                             {index_of_child: {'Resolution': 'Aleph Company Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Street Address': str(schema['properties']['addressEntity'][0]['properties']['full'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Address'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Person Address', 'Notes': ''}}])
+                        for country_code in schema['collection']['countries']:
+                            return_result.append(
+                                [{'Country Name': str(pycountry.countries.get(alpha_2=country_code).name),
+                                  'Notes': '',
+                                  'Entity Type': 'Country'},
+                                 {index_of_child_of_child: {'Resolution': 'Country', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph Collection ID'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Entity Search', 'Notes': ''}}])
+                except (TypeError, KeyError):
+                    # print(repr(e))
+                    continue
+        return return_result

Modules/Aleph/GetCollectionByID.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+
+class GetCollectionByID:
+    name = "Get Collections By ID"
+    category = "Aleph OCCRP"
+    description = "Find information about Collections and their IDs"
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase. Person, Address, Phone Number, Email Address, Country, Bank Account'}
+    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                url = f"https://aleph.occrp.org/api/2/collections/{primary_field}"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            if response['statistics']['names'].get('values') is not None:
+                nameKeys = list(response['statistics']['names'].get('values').keys())
+                for nameKey in nameKeys:
+                    returnResults.append([{'Full Name': str(nameKey),
+                                           'Entity Type': 'Person'},
+                                          {uid: {'Resolution': 'Person Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['addresses'].get('values') is not None:
+                addressKeys = list(response['statistics']['addresses'].get('values').keys())
+                for addressKey in addressKeys:
+                    returnResults.append([{'Street Address': str(addressKey),
+                                           'Entity Type': 'Address'},
+                                          {uid: {'Resolution': 'Address Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['phones'].get('values') is not None:
+                phoneKeys = list(response['statistics']['phones'].get('values').keys())
+                for phoneKey in phoneKeys:
+                    returnResults.append([{'Phone Number': str(phoneKey),
+                                           'Entity Type': 'Phone Number'},
+                                          {uid: {'Resolution': 'Phone Number Entity',
+                                                 'Notes': ''}}])
+
+            if response['statistics']['emails'].get('values') is not None:
+                emailKeys = list(response['statistics']['emails'].get('values').keys())
+                for emailKey in emailKeys:
+                    returnResults.append([{'Email Address': str(emailKey),
+                                           'Entity Type': 'Email Address'},
+                                          {uid: {'Resolution': 'Email Address Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['countries'].get('values') is not None:
+                countriesKeys = list(response['statistics']['countries'].get('values').keys())
+                for countriesKey in countriesKeys:
+                    returnResults.append([{'Country Name': str(countriesKey),
+                                           'Entity Type': 'Country'},
+                                          {uid: {'Resolution': 'Country Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['languages'].get('values') is not None:
+                languagesKeys = list(response['statistics']['languages'].get('values').keys())
+                for languagesKey in languagesKeys:
+                    returnResults.append([{'Phrase': str(languagesKey),
+                                           'Entity Type': 'Phrase'},
+                                          {uid: {'Resolution': 'Language Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['ibans'].get('values') is not None:
+                ibansKeys = list(response['statistics']['ibans'].get('values').keys())
+                for ibansKey in ibansKeys:
+                    returnResults.append([{'Account Number': str(ibansKey),
+                                           'Entity Type': 'Bank Account'},
+                                          {uid: {'Resolution': 'IBAN Entity',
+                                                 'Notes': ''}}])
+        return returnResults

Modules/Aleph/GetCollectionsInfo.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+
+class GetCollectionsInfo:
+    name = "Get Collections Info"
+    category = "Aleph OCCRP"
+    description = "Find information about Collections and their IDs"
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase, Aleph ID'}
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.',
+                                  'type': 'String',
+                                  'default': '1'},
+                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "The value for parameter 'Max Results' is not a valid integer."
+        if maxResults <= 0:
+            return []
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                url = f"https://aleph.occrp.org/api/2/collections?offset=0&limit=300&page"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            collections = response['results'][:maxResults]
+
+            for collection in collections:
+                index_of_child = len(returnResults)
+                returnResults.append([{'Phrase': collection['label'],
+                                       'Notes': str(collection.get('summary')),
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aleph Collection Name',
+                                             'Notes': ''}}])
+
+                returnResults.append([{'ID': collection['id'],
+                                       'Entity Type': 'Aleph ID'},
+                                      {index_of_child: {'Resolution': 'Aleph Collection ID',
+                                                        'Notes': ''}}])
+        return returnResults

Modules/Aleph/GetSimilarEntities.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+
+class GetSimilarEntities:
+    name = "Get Similar Entities"
+    category = "Aleph OCCRP"
+    description = "Find information about similar entities"
+    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
+    resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID'}
+    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        import pycountry
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                url = f"https://aleph.occrp.org/api/2/entities/{primary_field}/similar"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            entities = response['results']
+            for schema in entities:
+                if schema['entity']['schema'] == 'Person':
+                    index_of_child = len(returnResults)
+                    returnResults.append([{'Full Name': ' '.join(map(str, schema['entity']['properties']['name'])),
+                                           'Gender': ' '.join(map(str, schema['entity']['properties']['gender'])),
+                                           'Notes': ' '.join(map(str, schema['entity']['properties']['legalForm'])),
+                                           'Entity Type': 'Person'},
+                                          {uid: {'Resolution': 'Person Entity',
+                                                 'Notes': ''}}])
+                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
+                    returnResults.append([{'Street Address': schema['entity']['properties']['addressEntity'][0]
+                                           ['properties']['full'][0],
+                                           'Postal Code': schema['entity']['properties']['addressEntity'][0]
+                                           ['properties']['postalCode'][0],
+                                           'Country': country,
+                                           'Entity Type': 'Address'},
+                                          {index_of_child: {'Resolution': 'Address',
+                                                            'Notes': ''}}])
+                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
+                                           'Notes': schema['entity']['collection']['summary'],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Location in Database',
+                                                            'Notes': ''}}])
+
+                    returnResults.append([{'ID': schema['entity']['id'],
+                                           'Entity Type': 'Aleph ID'},
+                                          {index_of_child: {'Resolution': 'ID in Database',
+                                                            'Notes': ''}}])
+
+                elif schema['entity']['schema'] == 'Company':
+                    index_of_child = len(returnResults)
+                    returnResults.append([{'Company Name': ' '.join(map(str, schema['entity']['properties']['name'])),
+                                           'Notes': str(schema['entity']['properties']['status']),
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'Company Entity',
+                                                 'Notes': ''}}])
+                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
+                    returnResults.append([{'Street Address': str(schema['entity']['properties'].get('address')),
+                                           'Country': country,
+                                           'Entity Type': 'Address'},
+                                          {index_of_child: {'Resolution': 'Address',
+                                                            'Notes': ''}}])
+                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
+                                           'Notes': schema['entity']['collection']['summary'],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Location in Database',
+                                                            'Notes': ''}}])
+
+                    returnResults.append([{'ID': schema['entity']['id'],
+                                           'Entity Type': 'Aleph ID'},
+                                          {index_of_child: {'Resolution': 'ID in Database',
+                                                            'Notes': ''}}])
+        return returnResults

Modules/Aleph/requirements.txt

@@ -0,0 +1,3 @@
+requests
+pycountry
+requests-futures

Modules/Amass/Amass_Domain.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+# User needs to be in docker group or to have root privileges
+
+class Amass_Domain:
+    name = "Amass Domain Scan"
+    category = "Network Infrastructure"
+    description = "Find information about a particular domain. Requires Docker to be installed."
+    originTypes = {'Domain'}
+    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain', 'IPv6 Address'}
+    parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://virustotal.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'AlienVault API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://otx.alienvault.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://app.binaryedge.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'C99 API Key': {'description': 'Enter your api key under your profile after'
+                                                 ' signing up on https://c99.nl.',
+                                  'type': 'String',
+                                  'value': 'None',
+                                  'global': True,
+                                  'default': 'None'},
+                  'Censys API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://censys.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Chaos API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://chaos.projectdiscovery.io.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://cloudflare.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'DNSDB API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://dnsdb.info.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'GitHub API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://github.com.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Hunter API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://hunter.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
+                                                         ' signing up on https://ipinfo.io.',
+                                          'type': 'String',
+                                          'value': 'None',
+                                          'global': True,
+                                          'default': 'None'},
+                  'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://networksdb.io.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
+                                                          ' signing up on https://passivetotal.com .',
+                                           'type': 'String',
+                                           'value': 'None',
+                                           'global': True,
+                                           'default': 'None'},
+                  'ReconDev API Key': {'description': 'Enter your api key under your profile after'
+                                                      ' signing up on https://recon.dev.',
+                                       'type': 'String',
+                                       'value': 'None',
+                                       'global': True,
+                                       'default': 'None'},
+                  'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
+                                                            ' signing up on https://securitytrails.com.',
+                                             'type': 'String',
+                                             'value': 'None',
+                                             'global': True,
+                                             'default': 'None'},
+                  'Shodan API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://shodan.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Spyse API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://spyse.com.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://threatbook.cn.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'Umbrella API Key': {'description': 'Enter your api key under your profile after'
+                                                      ' signing up on https://umbrella.cisco.com.',
+                                       'type': 'String',
+                                       'value': 'None',
+                                       'global': True,
+                                       'default': 'None'},
+                  'URLScan API Key': {'description': 'Enter your api key under your profile after'
+                                                     ' signing up on https://urlscan.io.',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
+                                                         ' signing up on https://whoisxmlapi.com.',
+                                          'type': 'String',
+                                          'value': 'None',
+                                          'global': True,
+                                          'default': 'None'},
+                  'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://zetalytics.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
+                                                     'in between',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
+                                                        'in between. Obtain them at https://developer.facebook.com',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
+                                                     'in between. Obtain them at https://developer.twitter.com',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'ReconDev.free API Key': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'},
+                  'ReconDev.paid API Key': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import json
+        from ipaddress import ip_address, IPv4Address, IPv6Address
+        import docker
+        import tempfile
+        from docker.errors import APIError
+
+        return_result = []
+        # Generate Config as a temporary file:
+        with tempfile.TemporaryDirectory() as tempDir:
+            tempPath = Path(tempDir).absolute()
+            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                 suffix='Config',
+                                                 dir=tempPath)
+            config.write("share = true\n")
+            config.write("[scope]\n")
+            config.write("port = 80\n")
+            config.write("port = 443\n")
+            config.write("[data_sources]\n")
+            config.write("minimum_ttl = 1440\n")
+            for parameter in self.parameters:
+                if parameters[f'{parameter}'] != 'None':
+                    field1 = f"[data_sources.{parameter}]"
+                    field2 = f"[data_sources.{parameter}.Credentials]"
+                    if parameter == "ZoomEye":
+                        username, password = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"username = {username}\n")
+                        config.write(f"password = {password}\n")
+                    elif parameter == "FacebookCT":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.app1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "Twitter":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.account1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "ReconDev.paid":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.paid\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    elif parameter == "ReconDev.free":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.free\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    else:
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+            path_to_config = Path(config.name).name
+            config.seek(0)
+            for entity in entityJsonList:
+                primary_field = entity["Domain Name"].strip()
+                try:
+                    client = docker.from_env()
+                    container = client.containers.run("caffix/amass:latest",
+                                                      f"enum -src -d {primary_field} "
+                                                      f"-config /.config/amass/{path_to_config}",
+                                                      volumes={
+                                                          str(tempPath): {'bind': '/.config/amass',
+                                                                          'mode': 'rw'}},
+                                                      remove=True)
+                    jsonFile = tempPath / 'amass.json'
+                    jsonContents = ""
+                    if jsonFile.exists():
+                        with open(jsonFile, 'r') as jsonFileHandler:
+                            jsonContents = jsonFileHandler.read()
+                    client.close()
+                except (APIError, docker.errors.ContainerError) as error:
+                    return "Something happened to the docker container - Cannot continue: " + str(error)
+                uid = entity['uid']
+                for dictionary in jsonContents.splitlines():
+                    index_of_child = len(return_result)
+                    line_dictionary = json.loads(dictionary)
+                    size = len(line_dictionary['addresses'])
+                    return_result.append([{'Domain Name': str(line_dictionary['name']),
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
+                    for ip in range(size):
+                        if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
+                            return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
+                                                   'Entity Type': 'IP Address'},
+                                                  {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
+                        elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
+                            return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
+                                                   'Entity Type': 'IPv6 Address'},
+                                                  {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
+                        return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
+                                               'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
+                                               'Entity Type': 'Autonomous System'},
+                                              {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
+                        return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
+                                               'Entity Type': 'Phrase'},
+                                              {index_of_child: {'Resolution': 'Amass Domain Scan Description',
+                                                                'Notes': ''}}])
+            config.close()
+        return return_result

Modules/Amass/Amass_Intel.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+# User needs to be in docker group or have root privileges
+
+class Amass_Intel:
+    name = "Amass Intel Scan"
+    category = "Network Infrastructure"
+    description = "Find information about a particular domain. Requires Docker to be installed."
+    originTypes = {'Domain', 'IP Address', 'Autonomous System'}
+    resultTypes = {'Domain'}
+    parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://virustotal.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'AlienVault API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://otx.alienvault.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://app.binaryedge.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'C99 API Key': {'description': 'Enter your api key under your profile after'
+                                                 ' signing up on https://c99.nl.',
+                                  'type': 'String',
+                                  'value': 'None',
+                                  'global': True,
+                                  'default': 'None'},
+                  'Censys API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://censys.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Chaos API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://chaos.projectdiscovery.io.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://cloudflare.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'DNSDB API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://dnsdb.info.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'GitHub API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://github.com.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Hunter API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://hunter.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
+                                                         ' signing up on https://ipinfo.io.',
+                                          'type': 'String',
+                                          'value': 'None',
+                                          'global': True,
+                                          'default': 'None'},
+                  'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://networksdb.io.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
+                                                          ' signing up on https://passivetotal.com .',
+                                           'type': 'String',
+                                           'value': 'None',
+                                           'global': True,
+                                           'default': 'None'},
+                  'ReconDev API Key': {'description': 'Enter your api key under your profile after'
+                                                      ' signing up on https://recon.dev.',
+                                       'type': 'String',
+                                       'value': 'None',
+                                       'global': True,
+                                       'default': 'None'},
+                  'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
+                                                            ' signing up on https://securitytrails.com.',
+                                             'type': 'String',
+                                             'value': 'None',
+                                             'global': True,
+                                             'default': 'None'},
+                  'Shodan API Key': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://shodan.io.',
+                                     'type': 'String',
+                                     'value': 'None',
+                                     'global': True,
+                                     'default': 'None'},
+                  'Spyse API Key': {'description': 'Enter your api key under your profile after'
+                                                   ' signing up on https://spyse.com.',
+                                    'type': 'String',
+                                    'value': 'None',
+                                    'global': True,
+                                    'default': 'None'},
+                  'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://threatbook.cn.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'Umbrella API Key': {'description': 'Enter your api key under your profile after'
+                                                      ' signing up on https://umbrella.cisco.com.',
+                                       'type': 'String',
+                                       'value': 'None',
+                                       'global': True,
+                                       'default': 'None'},
+                  'URLScan API Key': {'description': 'Enter your api key under your profile after'
+                                                     ' signing up on https://urlscan.io.',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
+                                                         ' signing up on https://whoisxmlapi.com.',
+                                          'type': 'String',
+                                          'value': 'None',
+                                          'global': True,
+                                          'default': 'None'},
+                  'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
+                                                        ' signing up on https://zetalytics.com.',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
+                                                     'in between',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
+                                                        'in between. Obtain them at https://developer.facebook.com',
+                                         'type': 'String',
+                                         'value': 'None',
+                                         'global': True,
+                                         'default': 'None'},
+                  'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
+                                                     'in between. Obtain them at https://developer.twitter.com',
+                                      'type': 'String',
+                                      'value': 'None',
+                                      'global': True,
+                                      'default': 'None'},
+                  'ReconDev.free API Key': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'},
+                  'ReconDev.paid API Key': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None',
+                      'global': True,
+                      'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import json
+        from ipaddress import ip_address, IPv4Address, IPv6Address
+        import docker
+        import tempfile
+        from docker.errors import APIError
+
+        return_result = []
+        # Generate Config as a temporary file:
+        with tempfile.TemporaryDirectory() as tempDir:
+            tempPath = Path(tempDir).absolute()
+            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                 suffix='Config',
+                                                 dir=tempPath)
+            config.write("share = true\n")
+            config.write("[scope]\n")
+            config.write("port = 80\n")
+            config.write("port = 443\n")
+            config.write("[data_sources]\n")
+            config.write("minimum_ttl = 1440\n")
+            for parameter in parameters:
+                if parameters[f'{parameter}'] != 'None':
+                    field1 = f"[data_sources.{parameter}]"
+                    field2 = f"[data_sources.{parameter}.Credentials]"
+                    if parameter == "ZoomEye":
+                        username, password = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"username = {username}\n")
+                        config.write(f"password = {password}\n")
+                    elif parameter == "FacebookCT":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.app1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "Twitter":
+                        field3, secret = parameters[parameter].split(' ', 1)
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.account1\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                        config.write(f"secret = {secret}\n")
+                    elif parameter == "ReconDev.paid":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.paid\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    elif parameter == "ReconDev.free":
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"[data_sources.{parameter}.free\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+                    else:
+                        field3 = parameters[f'{parameter}']
+                        config.write(f"{field1}\n")
+                        config.write(f"{field2}\n")
+                        config.write(f"apikey = \"{field3}\"\n")
+            path_to_config = "/" + Path(config.name).name
+            config.seek(0)
+            for entity in entityJsonList:
+                primary_field = entity[list(entity)[1]].strip()
+                try:
+                    client = docker.from_env()
+                    if entity['Entity Type'] == "Domain":
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -whois -d {primary_field} -config /.config/amass"
+                                                          f"{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    elif entity['Entity Type'] == "IP Address":
+                        try:
+                            ip_address(primary_field)
+                        except ValueError:
+                            return "The Entity Provided isn't a valid IP Address"
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -addr {primary_field} -config "
+                                                          f"/.config/amass{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    elif entity['Entity Type'] == "Autonomous System":
+                        if primary_field.startswith('AS'):
+                            primary_field = primary_field[2:]
+                        container = client.containers.run("caffix/amass:latest",
+                                                          f"intel -asn {primary_field}"
+                                                          f" -config /.config/amass{path_to_config}",
+                                                          volumes={
+                                                              str(tempPath): {'bind': '/.config/amass',
+                                                                              'mode': 'rw'}},
+                                                          remove=True)
+                    textFile = tempPath / 'amass.txt'
+                    textContents = ""
+                    if textFile.exists():
+                        with open(textFile, 'r') as textFileHandler:
+                            textContents = textFileHandler.read()
+
+                    client.close()
+                except (APIError, docker.errors.ContainerError) as error:
+                    return "Something happened to the docker container - Cannot continue: " + str(error)
+                uid = entity['uid']
+                for newDomain in textContents.splitlines():
+                    return_result.append([{'Domain Name': newDomain.strip(),
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
+
+            config.close()
+        return return_result

Modules/Amass/requirements.txt

@@ -0,0 +1 @@
+docker

Modules/BigMatch/BigMatch.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+
+class BigMatch:
+    name = "BigMatch Search"
+    category = "Secrets & Leaks"
+    description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
+    originTypes = {"Image", "Document", "Archive"}
+    resultTypes = {'Website'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+
+        return_result = []
+
+        url = "https://bigmatch.rev.ng/static/index.html"
+        failString = 'Too many strings in binary?'
+        successString = 'Results:'
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+
+            for entity in entityJsonList:
+                uid = entity['uid']
+                file_path = Path(parameters['Project Files Directory']) / entity["File Path"]
+                file_path = file_path.absolute()
+                if not (file_path.exists() and file_path.is_file()):
+                    continue
+                page.wait_for_timeout(3000)
+
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        inputLocator = page.locator("input")
+                        inputLocator.set_input_files([str(file_path)])
+                        page.wait_for_timeout(3000)
+                        soup = BeautifulSoup(page.content(), 'lxml')
+                        soupText = soup.get_text()
+                        while (failString not in soupText) and (successString not in soupText):
+                            page.wait_for_timeout(1000)
+                            soup = BeautifulSoup(page.content(), 'lxml')
+                            soupText = soup.get_text()
+                        if failString in soupText:
+                            return []
+                        for link in soup.find_all('a'):
+                            potentialLink = link.get('href', None)
+                            if potentialLink is not None:
+                                if 'github' in potentialLink:
+                                    return_result.append([{'URL': potentialLink, 'Entity Type': 'Website'},
+                                                          {uid: {'Resolution': 'BigMatch Github Link', 'Notes': ''}}])
+
+                        break
+                    except TimeoutError:
+                        pass
+                    except Error:
+                        break
+            page.close()
+            browser.close()
+        return return_result

Modules/BigMatch/requirements.txt

@@ -0,0 +1,2 @@
+bs4
+playwright

Modules/BinaryEdge/BinaryEdgeHost.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+
+class BinaryEdgeHost:
+    name = "BinaryEdge Host Query"
+    category = "Network Infrastructure"
+    description = "Get information about a host from BinaryEdge."
+    originTypes = {"IP Address", "IPv6 Address"}
+    resultTypes = {'Port'}
+    parameters = {'BinaryEdge API Key': {'description': "Enter your BinaryEdge API key. Sign up for one at "
+                                                        "https://www.binaryedge.io/",
+                                         'type': 'String',
+                                         'value': '',
+                                         'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+
+        baseURL = 'https://api.binaryedge.io/v2/query/ip/'
+        requestHeaders = {'X-Key': parameters['BinaryEdge API Key'].strip()}
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if entity['Entity Type'] == 'IP Address':
+                primaryField = entity['IP Address']
+            elif entity['Entity Type'] == 'IPv6 Address':
+                primaryField = entity['IPv6 Address']
+            else:
+                continue
+            infoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
+            statusCode = infoRequest.status_code
+
+            if statusCode == 401:
+                return "The BinaryEdge API key provided is not valid."
+            elif statusCode == 403:
+                return "The BinaryEdge API key provided does not have permission to access this resource."
+            elif statusCode != 200:
+                continue
+            requestContent = json.loads(infoRequest.content)
+
+            for event in requestContent['events']:
+                for result in event['results']:
+                    originDetails = result['origin']
+                    targetDetails = result['target']
+                    resultDetails = result['result']
+                    if 'state' not in resultDetails['data']:
+                        # Discard return result if it doesn't actually give us useful info about the state of the port.
+                        # This happens in cases where the API returns stuff like the ciphers used in an SSH service.
+                        #   There seems to always be a result with the simple port info, so we will use that one.
+                        continue
+
+                    returnResults.append([{'Port': targetDetails['ip'] + ':' + str(targetDetails['port']) + ':' +
+                                                   targetDetails['protocol'],
+                                           'State': resultDetails['data']['state']['state'],
+                                           'Banner': resultDetails['data']['service'].get('banner', 'N/A'),
+                                           'Product': resultDetails['data']['service'].get('product', 'Unknown'),
+                                           'Entity Type': 'Port'},
+                                          {uid: {'Resolution': 'BinaryEdge Scan Timestamp: ' + str(originDetails['ts']),
+                                                 'Notes': ''}}])
+
+        return returnResults

Modules/BinaryEdge/requirements.txt

@@ -0,0 +1 @@
+requests

Modules/BreachData/HIBP.xml

@@ -0,0 +1,34 @@
+<HIBP>
+    <Data_Breach>
+        <Attributes>
+            <Attribute default="Breach Name" check="String" primary="True">Breach Name</Attribute>
+            <Attribute default="Breach Title" check="String" primary="False">Breach Title</Attribute>
+            <Attribute default="Breach Domain" check="String" primary="False">Breach Domain</Attribute>
+            <Attribute default="0" check="Numbers" primary="False">Breach Pwn Count</Attribute>
+            <Attribute default="Breach Description" check="String" primary="False">Breach Description</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Sensitive</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Verified</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Fabricated</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Retired</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Spam List</Attribute>
+            <Attribute default="False" check="String" primary="False">Breach Is Malware</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Breach Added Date</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Breach Modified Date</Attribute>
+        </Attributes>
+        <Icon>
+            Default.svg
+        </Icon>
+    </Data_Breach>
+    <Paste_Data_Leak>
+        <Attributes>
+            <Attribute default="Paste Identifier" check="String" primary="True">Paste Identifier</Attribute>
+            <Attribute default="Paste Title" check="String" primary="False">Paste Title</Attribute>
+            <Attribute default="Paste Source" check="String" primary="False">Paste Source</Attribute>
+            <Attribute default="Paste ID" check="String" primary="False">Paste ID</Attribute>
+            <Attribute default="0" check="Numbers" primary="False">Paste Email Count</Attribute>
+        </Attributes>
+        <Icon>
+            Default.svg
+        </Icon>
+    </Paste_Data_Leak>
+</HIBP>

Modules/BreachData/HIBPBreachToDomain.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+class HIBPBreachToDomain:
+
+    name = "HIBP Breach To Domain"
+    category = "Leaked Data"
+    description = "Get the domain of the primary website that a data breach occurred on."
+    originTypes = {'Data Breach'}
+    resultTypes = {'Domain'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            domainMaybe = entity.get('Breach Domain')
+            if isinstance(domainMaybe, str):
+                if domainMaybe.strip() != '':
+                    returnResults.append([{'Domain Name': domainMaybe,
+                                           'Entity Type': 'Domain'},
+                                          {entity['uid']: {'Resolution': 'Data Breach to Domain',
+                                                           'Notes': ''}}])
+
+        return returnResults

Modules/BreachData/HaveIBeenPwnedBreachDomains.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedBreachDomains:
+    name = "HIBP Breach Domain Lookup"
+    category = "Leaked Data"
+    description = "Find breaches associated with a specified domain."
+    originTypes = {'Domain'}
+    resultTypes = {'Data Breach'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+
+        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
+        from PySide6.QtGui import QImage
+
+        baseURL = "https://haveibeenpwned.com/api/v3/breaches?domain="
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            primaryField = entity['Domain Name']
+            breachInfoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 200:
+                breachContent = json.loads(breachInfoRequest.content)
+
+                for breach in breachContent:
+                    try:
+                        breachLogoIconRequest = requests.get(breach['LogoPath'])
+                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
+                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
+                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
+
+                        # Rotate the breach domain logo upside down
+                        breachIconImageRotated = breachIconImageScaled.mirrored()
+
+                        breachIconByteArrayFin = QByteArray()
+                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
+                        breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+                        breachIconImageRotated.save(breachImageBuffer, "PNG")
+                        breachImageBuffer.close()
+                    except Exception:
+                        breachIconByteArrayFin = None
+
+                    # If Breach Date is None, then default to entity creation date.
+                    returnResults.append([{'Breach Name': breach['Name'],
+                                           'Breach Title': breach['Title'],
+                                           'Breach Domain': breach['Domain'],
+                                           'Breach Pwn Count': str(breach['PwnCount']),
+                                           'Breach Description': breach['Description'],
+                                           'Breach Is Sensitive': str(breach['IsSensitive']),
+                                           'Breach Is Verified': str(breach['IsVerified']),
+                                           'Breach Is Fabricated': str(breach['IsFabricated']),
+                                           'Breach Is Retired': str(breach['IsRetired']),
+                                           'Breach Is Spam List': str(breach['IsSpamList']),
+                                           'Breach Is Malware': str(breach['IsMalware']),
+                                           'Breach Added Date': breach['AddedDate'],
+                                           'Breach Modified Date': breach['ModifiedDate'],
+                                           'Entity Type': 'Data Breach',
+                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
+                                           'Date Created': breach['BreachDate']},
+                                          {entity['uid']: {'Resolution': 'Contained in Breach',
+                                                           'Notes': ''}}])
+            elif statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedBreaches.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedBreaches:
+    name = "HIBP Breach Lookup"
+    category = "Leaked Data"
+    description = "Find all breaches that an account has been involved in. Note that Date Created for breaches is an " \
+                  "estimate."
+    originTypes = {'Email Address', 'Phone Number'}
+    resultTypes = {'Data Breach'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+        from urllib.parse import quote_plus
+
+        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
+        from PySide6.QtGui import QImage
+
+        baseURL = "https://haveibeenpwned.com/api/v3/breachedaccount/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            primaryField = entity[list(entity)[1]]
+            breachInfoRequest = requests.get(baseURL + quote_plus(primaryField) + '?truncateResponse=false',
+                                             headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 200:
+                breachContent = json.loads(breachInfoRequest.content)
+
+                for breach in breachContent:
+                    try:
+                        breachLogoIconRequest = requests.get(breach['LogoPath'])
+                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
+                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
+                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
+
+                        # Rotate the breach domain logo upside down
+                        breachIconImageRotated = breachIconImageScaled.mirrored()
+
+                        breachIconByteArrayFin = QByteArray()
+                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
+                        breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+                        breachIconImageRotated.save(breachImageBuffer, "PNG")
+                        breachImageBuffer.close()
+                    except Exception:
+                        breachIconByteArrayFin = None
+
+                    # If Breach Date is None, then default to entity creation date.
+                    returnResults.append([{'Breach Name': breach['Name'],
+                                           'Breach Title': breach['Title'],
+                                           'Breach Domain': breach['Domain'],
+                                           'Breach Pwn Count': str(breach['PwnCount']),
+                                           'Breach Description': breach['Description'],
+                                           'Breach Is Sensitive': str(breach['IsSensitive']),
+                                           'Breach Is Verified': str(breach['IsVerified']),
+                                           'Breach Is Fabricated': str(breach['IsFabricated']),
+                                           'Breach Is Retired': str(breach['IsRetired']),
+                                           'Breach Is Spam List': str(breach['IsSpamList']),
+                                           'Breach Is Malware': str(breach['IsMalware']),
+                                           'Breach Added Date': breach['AddedDate'],
+                                           'Breach Modified Date': breach['ModifiedDate'],
+                                           'Entity Type': 'Data Breach',
+                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
+                                           'Date Created': breach['BreachDate']},
+                                          {entity['uid']: {'Resolution': 'Contained in Breach',
+                                                           'Notes': ''}}])
+            elif statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPassword.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPassword:
+    name = "HIBP Password Lookup"
+    category = "Leaked Data"
+    description = "Check whether the given password was found in breaches."
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from time import sleep
+        from hashlib import sha1
+
+        baseURL = "https://api.pwnedpasswords.com/range/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+
+            primaryField = sha1(entity[list(entity)[1]].encode('utf-8')).hexdigest().upper()
+            hashPrefix = primaryField[:5]
+            hashSuffix = primaryField[5:]
+
+            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 200:
+                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
+
+                for password in pwnedPasswordContent:
+                    if hashSuffix in password:
+                        returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
+                                                         f"in breach data.",
+                                               'Entity Type': 'Phrase'},
+                                              {entity['uid']: {'Resolution': 'Pwned Password',
+                                                               'Notes': ''}}])
+                        break
+            elif statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPasswordHash.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPasswordHash:
+    name = "HIBP Password Hash Lookup"
+    category = "Leaked Data"
+    description = "Check whether the given password hash was found in breaches."
+    originTypes = {'Hash', 'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from time import sleep
+
+        baseURL = "https://api.pwnedpasswords.com/range/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+
+            primaryField = entity[list(entity)[1]].upper()
+            if len(primaryField) != 40:
+                continue
+            hashPrefix = primaryField[:5]
+            hashSuffix = primaryField[5:]
+
+            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
+            statusCode = breachInfoRequest.status_code
+            if statusCode == 200:
+                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
+
+                for password in pwnedPasswordContent:
+                    if hashSuffix in password:
+                        returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
+                                                         f"in breach data.",
+                                               'Entity Type': 'Phrase'},
+                                              {entity['uid']: {'Resolution': 'Pwned Password',
+                                                               'Notes': ''}}])
+                        break
+            elif statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/HaveIBeenPwnedPastes.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+
+class HaveIBeenPwnedPastes:
+    name = "HIBP Paste Lookup"
+    category = "Leaked Data"
+    description = "Find all pastes that an account has been involved in."
+    originTypes = {'Email Address'}
+    resultTypes = {'Paste Data Leak'}
+    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
+                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
+                                   'type': 'String',
+                                   'value': '',
+                                   'global': True,
+                                   'default': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import json
+        from time import sleep
+        from urllib.parse import quote_plus
+
+        baseURL = "https://haveibeenpwned.com/api/v3/pasteaccount/"
+        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
+
+        returnResults = []
+
+        count = 0
+        while count < len(entityJsonList):
+            entity = entityJsonList[count]
+            emailAddress = entity['Email Address']
+            pasteInfoRequest = requests.get(baseURL + quote_plus(emailAddress), headers=requestHeaders)
+            statusCode = pasteInfoRequest.status_code
+            if statusCode == 200:
+                pasteContent = json.loads(pasteInfoRequest.content)
+
+                for paste in pasteContent:
+                    pasteID = paste['Id']
+                    pasteSource = paste['Source']
+
+                    # If Paste Date is None, then default to entity creation date.
+                    returnResults.append([{'Paste Identifier': f'{pasteSource} | {pasteID}',
+                                           'Paste Title': paste['Title'],
+                                           'Paste Source': pasteSource,
+                                           'Paste ID': pasteID,
+                                           'Paste Email Count': str(paste['EmailCount']),
+                                           'Entity Type': 'Paste Data Leak',
+                                           'Date Created': paste['Date']},
+                                          {entity['uid']: {'Resolution': 'Contained in Paste',
+                                                           'Notes': ''}}])
+
+            elif statusCode == 401:
+                return "The HIBP API Key provided is invalid."
+            elif statusCode == 429:
+                sleep(2)
+                continue
+            elif statusCode == 503:
+                return "The HIBP Service is unavailable."
+            sleep(1.7)
+            count += 1
+        return returnResults

Modules/BreachData/requirements.txt

@@ -0,0 +1 @@
+requests