Small fixes to the build scripts.

Fixed a bug with cookies when importing tabs from Firefox.

Fixed a bug with creating duplicate nodes when importing browser tabs.

Added timestamps on the screenshots taken of the websites when importing tabs from the browser.

CONTRIBUTING

@@ -0,0 +1,3 @@
+By contributing to this software, the contributor hereby grants to AccentuSoft and to recipients of software distributed by AccentuSoft a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer all or parts of this software, where such license applies only to those patent claims licensable by the contributor that are necessarily infringed by the contributor's contribution(s) alone or by combination of the contributor's contribution(s) with the all or parts of this software to which such contribution(s) was submitted.
+
+In non-legalese, this means that any contributions made to this software can be used by AccentuSoft and any other user for any purpose without concern that the original author will revoke permission to use their contribution, or take anyone to court. Make sure that all your contributions consist of things that you want to contribute to the project. In all cases, the project's license still applies.

Core/Entities/Devices.xml

@@ -0,0 +1,26 @@
+<Devices>
+    <Mobile_Phone>
+        <Attributes>
+            Device UID,Brand
+        </Attributes>
+        <Icon>
+            Mobile_Device.svg
+        </Icon>
+    </Mobile_Phone>
+    <Computer>
+        <Attributes>
+            Hostname,Brand
+        </Attributes>
+        <Icon>
+            Computer.svg
+        </Icon>
+    </Computer>
+    <Camera>
+        <Attributes>
+            Device UID,Brand
+        </Attributes>
+        <Icon>
+            Camera.svg
+        </Icon>
+    </Camera>
+</Devices>

Core/Entities/Financials.xml

@@ -0,0 +1,18 @@
+<Financials>
+    <Bank_Account>
+        <Attributes>
+            Account Number,Bank Name,Branch Number
+        </Attributes>
+        <Icon>
+            Bank_Account.svg
+        </Icon>
+    </Bank_Account>
+    <Currency>
+        <Attributes>
+            Amount,Currency Type
+        </Attributes>
+        <Icon>
+            Currency.svg
+        </Icon>
+    </Currency>
+</Financials>

Core/Entities/Groups.xml

@@ -0,0 +1,42 @@
+<Collectives>
+    <Company>
+        <Attributes>
+            Company Name,Registration Number
+        </Attributes>
+        <Icon>
+            Company.svg
+        </Icon>
+    </Company>
+    <Organization>
+        <Attributes>
+            Organization Name,Registration Number
+        </Attributes>
+        <Icon>
+            Organization.svg
+        </Icon>
+    </Organization>
+    <Cult>
+        <Attributes>
+            Cult Name,Ideology
+        </Attributes>
+        <Icon>
+            Cult.svg
+        </Icon>
+    </Cult>
+    <Terrorist_Group>
+        <Attributes>
+            Group Name,Ideology
+        </Attributes>
+        <Icon>
+            Terrorists.svg
+        </Icon>
+    </Terrorist_Group>
+    <Family>
+        <Attributes>
+            Family Surname
+        </Attributes>
+        <Icon>
+            Family.svg
+        </Icon>
+    </Family>
+</Collectives>

Core/Entities/Identifiers.xml

@@ -0,0 +1,34 @@
+<Identifiers>
+    <Identification_Number>
+        <Attributes>
+            ID Number
+        </Attributes>
+        <Icon>
+            ID_Number.svg
+        </Icon>
+    </Identification_Number>
+    <Passport_Number>
+        <Attributes>
+            Passport Number
+        </Attributes>
+        <Icon>
+            Passport.svg
+        </Icon>
+    </Passport_Number>
+    <Vehicle_Registration>
+        <Attributes>
+            Registration Number
+        </Attributes>
+        <Icon>
+            VehicleRegistration.svg
+        </Icon>
+    </Vehicle_Registration>
+    <Flight_Number>
+        <Attributes>
+            Flight Number,Airline
+        </Attributes>
+        <Icon>
+            Flight_Number.svg
+        </Icon>
+    </Flight_Number>
+</Identifiers>

Core/Entities/Individuals.xml

@@ -0,0 +1,58 @@
+<Individuals>
+  <Person>
+    <Attributes>
+      Full Name,Gender,Occupation,Date of Birth
+    </Attributes>
+    <Icon>
+      Person.svg
+    </Icon>
+  </Person>
+  <Politically_Exposed_Person>
+    <Attributes>
+      Full Name,Gender,Occupation,Date of Birth
+    </Attributes>
+    <Icon>
+      Person.svg
+    </Icon>
+  </Politically_Exposed_Person>
+  <Phrase>
+    <Attributes>
+      Phrase
+    </Attributes>
+    <Icon>
+      Phrase.svg
+    </Icon>
+  </Phrase>
+  <Sentiment>
+    <Attributes>
+      Sentiment
+    </Attributes>
+    <Icon>
+      Sentiment.svg
+    </Icon>
+  </Sentiment>
+  <Email_Address>
+    <Attributes>
+      Email Address
+    </Attributes>
+    <Icon>
+      Email.svg
+    </Icon>
+  </Email_Address>
+  <Phone_Number>
+    <Attributes>
+      Phone Number
+    </Attributes>
+    <Icon>
+      Phone_Number.svg
+    </Icon>
+  </Phone_Number>
+  <Date>
+    <Attributes>
+      Date
+    </Attributes>
+    <Icon>
+      Date.svg
+    </Icon>
+  </Date>
+</Individuals>

Core/Entities/Infrastructure.xml

@@ -0,0 +1,87 @@
+<Infrastructure>
+  <Operating_System>
+    <Attributes>
+      OS Name
+    </Attributes>
+    <Icon>
+      OperatingSystem.svg
+    </Icon>
+  </Operating_System>
+  <Network>
+    <Attributes>
+      IP Address,Range
+    </Attributes>
+    <Icon>
+      Network.svg
+    </Icon>
+  </Network>
+  <Website>
+    <Attributes>
+      URL
+    </Attributes>
+    <Icon>
+      Website.svg
+    </Icon>
+  </Website>
+  <Domain>
+    <Attributes>
+      Domain Name
+    </Attributes>
+    <Icon>
+      Domain.svg
+    </Icon>
+  </Domain>
+  <IP_Address>
+    <Attributes>
+      IP Address
+    </Attributes>
+    <Icon>
+      IP_Address.svg
+    </Icon>
+  </IP_Address>
+  <IPv6_Address>
+    <Attributes>
+      IPv6 Address
+    </Attributes>
+    <Icon>
+      IPv6_Address.svg
+    </Icon>
+</IPv6_Address>
+<Autonomous_System>
+    <Attributes>
+      AS Number,ASN Cidr
+    </Attributes>
+    <Icon>
+      AutonomousSystem.svg
+    </Icon>
+  </Autonomous_System>
+  <Port>
+    <Attributes>
+      Port
+    </Attributes>
+    <Icon>
+      Port.svg
+    </Icon>
+  </Port>
+  <MAC_Address>
+    <Attributes>
+      MAC Address
+    </Attributes>
+    <Icon>
+      MAC_Address.svg
+    </Icon>
+  </MAC_Address>
+  <Network>
+    <Attributes>
+      IP Address,Range
+    </Attributes>
+  </Network>
+  <Website_Infrastructure>
+    <Attributes>
+      Infrastructure
+    </Attributes>
+    <Icon>
+      WebInfrastructure.svg
+    </Icon>
+  </Website_Infrastructure>
+</Infrastructure>

Core/Entities/Materials.xml

@@ -0,0 +1,42 @@
+<Materials>
+  <Document>
+    <Attributes>
+      Document Name,File Path
+    </Attributes>
+    <Icon>
+      Document.svg
+    </Icon>
+  </Document>
+  <Image>
+    <Attributes>
+      Image Name,File Path
+    </Attributes>
+    <Icon>
+      Image.svg
+    </Icon>
+  </Image>
+  <Video>
+    <Attributes>
+      Video Name,File Path
+    </Attributes>
+    <Icon>
+      Video.svg
+    </Icon>
+  </Video>
+  <Archive>
+    <Attributes>
+      Archive Name,File Path
+    </Attributes>
+    <Icon>
+      Archive.svg
+    </Icon>
+  </Archive>
+  <Disk>
+    <Attributes>
+      Disk Name,File Path
+    </Attributes>
+    <Icon>
+      HardDisk.svg
+    </Icon>
+</Disk>
+</Materials>

Core/Entities/Meta.xml

@@ -0,0 +1,10 @@
+<Meta>
+  <EntityGroup>
+    <Attributes>
+      Group Name,Child UIDs
+    </Attributes>
+    <Icon>
+      GroupNode.svg
+    </Icon>
+  </EntityGroup>
+</Meta>

Core/Entities/Places.xml

@@ -0,0 +1,42 @@
+<Places>
+  <GeoCoordinates>
+    <Attributes>
+      Label,Latitude,Longitude
+    </Attributes>
+    <Icon>
+      Geolocation.svg
+    </Icon>
+  </GeoCoordinates>
+  <Address>
+    <Attributes>
+      Street Address,Locality,Postal Code,Country
+    </Attributes>
+    <Icon>
+      Address.svg
+    </Icon>
+  </Address>
+  <Country>
+    <Attributes>
+      Country Name
+    </Attributes>
+    <Icon>
+      Country.svg
+    </Icon>
+  </Country>
+  <City>
+    <Attributes>
+      City Name
+    </Attributes>
+    <Icon>
+      City.svg
+    </Icon>
+  </City>
+  <Circular_Area>
+    <Attributes>
+      Label,Latitude,Longitude,Radius
+    </Attributes>
+    <Icon>
+      GeoArea.svg
+    </Icon>
+  </Circular_Area>
+</Places>

Core/Entities/Social Media.xml

@@ -0,0 +1,18 @@
+<Social>
+    <Social_Media_Group>
+        <Attributes>
+            Group Name,Social Media Site
+        </Attributes>
+        <Icon>
+            Social_Media_Group.svg
+        </Icon>
+    </Social_Media_Group>
+    <Social_Media_Handle>
+        <Attributes>
+            User Name
+        </Attributes>
+        <Icon>
+            Social_Media_Account.svg
+        </Icon>
+    </Social_Media_Handle>
+</Social>

Core/Entities/Software.xml

@@ -0,0 +1,18 @@
+<Software>
+    <Application>
+        <Attributes>
+            Application Name
+        </Attributes>
+        <Icon>
+            Software.svg
+        </Icon>
+    </Application>
+    <Hash>
+        <Attributes>
+            Hash Value,Hash Algorithm
+        </Attributes>
+        <Icon>
+            Hash.svg
+        </Icon>
+    </Hash>
+</Software>

Core/EntityDB.py

@@ -0,0 +1,415 @@
+#!/usr/bin/env python3
+
+from shutil import move
+from pickle import load, dump
+from threading import Lock
+from pathlib import Path
+
+import networkx as nx
+
+
+class EntitiesDB:
+    """
+    This is a class that handles the addition and removal of entities and
+    links on a project-wide scale.
+    """
+
+    def __init__(self, mainWindow, messageHandler, resourceHandler):
+        self.messageHandler = messageHandler
+        self.resourceHandler = resourceHandler
+        self.mainWindow = mainWindow
+        self.dbLock = Lock()
+        self.database = None
+
+        self.loadDatabase()
+        self.resetTimeline()
+
+    def loadDatabase(self):
+        """
+        Load DiGraph from pickle file.
+        """
+        self.dbLock.acquire()
+        if self.database is not None:
+            self.save()
+        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.pkl")
+        self.messageHandler.debug('Opening Database at: ' + str(databaseFile))
+        try:
+            dbFile = open(databaseFile, "rb")
+            self.database = load(dbFile)
+            dbFile.close()
+            self.messageHandler.info('Loaded Local Entities Database.')
+        except FileNotFoundError:
+            self.messageHandler.info('Creating new Local Entities Database.')
+            self.database = nx.DiGraph()
+        finally:
+            self.dbLock.release()
+
+    def resetTimeline(self):
+        """
+        Reset the timeline on dockBarThree to reflect the current state of the entities database.
+        """
+        self.dbLock.acquire()
+        if self.database is not None:
+            self.mainWindow.resetTimeline(self.database)
+        self.dbLock.release()
+
+    def updateTimeline(self, node, added: bool, updateGraph: bool = True):
+        """
+        Update the timeline on dockBarThree to reflect the newest change of the entities database.
+        """
+        self.dbLock.acquire()
+        self.mainWindow.updateTimeline(node, added, updateGraph)
+        self.dbLock.release()
+
+    def save(self):
+        """
+        Saves the graph (pickles it too) to the specified file.
+        """
+
+        # Get the database file path again, in case it changed.
+        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.pkl")
+        if databaseFile is None:
+            raise ValueError('Database File is None, cannot save database.')
+        self.dbLock.acquire()
+        tmpSavePath = databaseFile.with_suffix(databaseFile.suffix + '.tmp')
+        dbFile = open(tmpSavePath, "wb")
+        dump(self.database, dbFile)
+        dbFile.close()
+        move(tmpSavePath, databaseFile)
+        self.messageHandler.info('Database Saved.')
+        self.dbLock.release()
+
+    def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True):
+        """
+        Adds the entity represented by the json dictionary to the database.
+        :param updateTimeline:
+        :param entJson:
+        :param fromServer:
+        :return:
+        """
+        self.dbLock.acquire()
+        returnValue = None
+
+        # Check if we're overwriting an existing entity
+        exists = None
+        if entJson.get('uid') is not None:
+            exists = self.getEntityNoLock(entJson.get('uid'))
+
+        entity = self.resourceHandler.getEntityJson(
+            entJson.get('Entity Type'),
+            entJson)
+
+        if entity is None:
+            self.dbLock.release()
+            return returnValue
+        # Use uid as key. Code is holdover from time where primary field == uid.
+        self.database.add_node(entity['uid'], **entity)
+        returnValue = entity
+        if exists:
+            # Update canvases if the node already exists.
+            self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
+        self.dbLock.release()
+        if not fromServer:
+            self.mainWindow.sendLocalDatabaseUpdateToServer(entity, True)
+        self.mainWindow.populateEntitiesWidget(returnValue, add=True)
+
+        if updateTimeline:
+            if exists is not None:
+                # Remove existing item before re-adding.
+                self.updateTimeline(exists, False, updateGraph=False)
+            self.updateTimeline(entity, True, updateGraph=True)
+
+        return returnValue
+
+    def addLink(self, linkJson: dict, fromServer=False):
+        """
+        Add a link between two entities in the database.
+
+        :param linkJson:
+        :param fromServer:
+        :return:
+        """
+        self.dbLock.acquire()
+        exists = self.isLinkNoLock(linkJson['uid'])
+        link = self.resourceHandler.getLinkJson(linkJson)
+        if link is None:
+            # This can technically be caused by a race condition if the user
+            #   either tries really hard or gets really unlucky.
+            # Caused by deleting a node faster than the link can be created.
+            self.messageHandler.error("Attempted to add Link with "
+                                      "no uid to database.")
+        else:
+            linkUID = link['uid']
+            self.database.add_edge(linkUID[0], linkUID[1], **link)
+            if exists:
+                # Update canvases if the link already exists.
+                self.mainWindow.updateLinkLabelsOnCanvases(linkUID[0] + linkUID[1], link['Resolution'])
+        self.dbLock.release()
+        if not fromServer:
+            self.mainWindow.sendLocalDatabaseUpdateToServer(link, True)
+        return link
+
+    def getEntity(self, uid: str):
+        """
+        Returns the attributes of the given entity uid as a dict.
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = self.database.nodes[uid]
+        except KeyError:
+            self.messageHandler.warning(
+                "Tried to get entity with nonexistent UID: " + uid)
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def getAllEntities(self):
+        """
+        Returns a list containing the Json representation of every entity in the database.
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = []
+            for node in self.database.nodes():
+                returnValue += [self.database.nodes[node]]
+        except KeyError:
+            self.messageHandler.error(
+                "Tried to get entity with nonexistent UID.")
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def getAllLinks(self):
+        """
+        Returns a list containing the Json representation of every link in the database.
+        :return:
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = []
+            for edge in self.database.edges():
+                returnValue += [self.database.edges[edge]]
+        except KeyError:
+            self.messageHandler.error(
+                "Tried to get link with nonexistent UID.")
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def getEntityNoLock(self, uid: str):
+        """
+        Returns the attributes of the given entity uid as a dict.
+
+        Does not lock, specifically meant for use by other functions in this
+        class.
+        """
+        returnValue = None
+        try:
+            returnValue = self.database.nodes[uid]
+        except KeyError:
+            pass
+        finally:
+            return returnValue
+
+    def getLink(self, uid):
+        """
+        Returns the attributes of the given link uid as a dict.
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = self.database.edges[uid]
+        except KeyError:
+            self.messageHandler.error(
+                "Tried to get link with nonexistent UID.")
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def removeEntity(self, uid: str, fromServer=False):
+        """
+        Removes the entity with the given uid, if it
+        exists.
+        """
+        self.dbLock.acquire()
+        ent = None
+        if self.isNodeNoLock(uid):
+            ent = self.getEntityNoLock(uid)
+            self.mainWindow.populateEntitiesWidget(ent, add=False)
+            self.database.remove_node(uid)
+        self.dbLock.release()
+        if ent is not None:
+            self.mainWindow.handleGroupNodeUpdateAfterEntityDeletion(uid)  # Blocking - locks the db.
+            if not fromServer:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(ent, False)
+            self.updateTimeline(ent, False)
+
+    def removeLink(self, uid, fromServer=False):
+        """
+        Removes the link with the given uid (in string or tuple form),
+        if it exists.
+        """
+        self.dbLock.acquire()
+        if self.isLinkNoLock(uid):
+            self.database.remove_edge(uid[0], uid[1])
+        self.dbLock.release()
+        if not fromServer:
+            self.mainWindow.sendLocalDatabaseUpdateToServer({"uid": uid}, False)
+
+    def doesEntityExist(self, primaryAttr: str):
+        """
+        Checks if an entity with the specified primary attribute exists.
+        """
+        self.dbLock.acquire()
+        result = False
+        for node in self.database.nodes():
+            details = self.database.nodes[node]
+            if details[list(details)[1]] == primaryAttr:
+                result = True
+                break
+        self.dbLock.release()
+        return result
+
+    def getLinkIfExists(self, uid):
+        """
+        Returns the attributes of the given link uid as a dict.
+        Does not create an error if the link does not exist.
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = self.database.edges[uid]
+        except KeyError:
+            pass
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def getIncomingLinks(self, uid: str):
+        """
+        Get all incoming edges for the given entity uid (primary attribute).
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        if self.isNodeNoLock(uid):
+            returnValue = self.database.in_edges(uid)
+        self.dbLock.release()
+        return returnValue
+
+    def getOutgoingLinks(self, uid: str):
+        """
+        Get all outgoing edges for the given entity uid (primary attribute).
+        """
+        self.dbLock.acquire()
+        returnValue = None
+        if self.isNodeNoLock(uid):
+            returnValue = self.database.out_edges(uid)
+        self.dbLock.release()
+        return returnValue
+
+    def isNode(self, uid: str):
+        """
+        Returns True if the uid (primary attribute) given exists as
+        an entity, and False otherwise.
+        """
+        self.dbLock.acquire()
+        returnValue = False
+        if self.database.nodes.get(uid) is not None:
+            returnValue = True
+        self.dbLock.release()
+        return returnValue
+
+    def isNodeNoLock(self, uid: str):
+        """
+        Returns True if the uid (primary attribute) given exists as
+        an entity, and False otherwise.
+
+        Used only in this class, as it does not lock.
+        """
+        if self.database.nodes.get(uid) is not None:
+            return True
+        return False
+
+    def isLink(self, uid: tuple):
+        """
+        Returns True if the uid given exists as a link, and False otherwise.
+        """
+        self.dbLock.acquire()
+        returnValue = False
+        item = self.database.edges.get(uid)
+        if item is not None:
+            returnValue = True
+        self.dbLock.release()
+        return returnValue
+
+    def isLinkNoLock(self, uid: tuple):
+        """
+        Returns True if the uid given exists as a link, and False otherwise.
+        
+        Used only in this class, as it does not lock.
+        """
+        if self.database.edges.get(uid) is not None:
+            return True
+        return False
+
+    def getEntityType(self, uid: str):
+        self.dbLock.acquire()
+        returnValue = None
+        try:
+            returnValue = self.getEntityNoLock(uid)['Entity Type']
+        except KeyError:
+            pass
+        finally:
+            self.dbLock.release()
+            return returnValue
+
+    def mergeDatabases(self, newDB_nodes: dict, newDB_edges: dict, fromServer=True):
+        """
+        Merges the existing database with the one provided.
+        
+        Overwrites older attributes with newer ones.
+        """
+        self.dbLock.acquire()
+        differenceGraph = nx.DiGraph()
+        # Note: If we ever receive a node without a 'Date Last Edited' field, ignore it.
+        differenceGraph.add_nodes_from([(x, newDB_nodes[x])
+                                        for x in newDB_nodes if (x not in self.database.nodes()) or
+                                        (
+                                                x in self.database.nodes() and
+                                                newDB_nodes[x].get('Date Last Edited') and
+                                                newDB_nodes[x]['Date Last Edited'] >
+                                                self.database.nodes[x]['Date Last Edited']
+                                        )
+                                        ])
+        differenceGraph.add_edges_from([(x, y, newDB_edges[(x, y)])
+                                        for x, y in newDB_edges if ((x, y) not in self.database.edges()) or
+                                        (
+                                                (x, y) in self.database.edges() and
+                                                newDB_edges[(x, y)].get('Date Last Edited') and
+                                                newDB_edges[(x, y)]['Date Last Edited'] >
+                                                self.database.edges[(x, y)]['Date Last Edited']
+                                        )
+                                        ])
+        if differenceGraph.number_of_nodes() > 0:
+            self.database = nx.compose(self.database, differenceGraph)
+            # Some nodes given by differenceGraph may be empty dicts, with an existing node's uid as the key.
+            for node in differenceGraph.nodes:
+                self.mainWindow.populateEntitiesWidget(self.database.nodes[node], add=True)
+
+            if not fromServer:
+                if self.mainWindow.FCOM.isConnected():
+                    # diffNew = nx.DiGraph()
+                    # diffNew.add_nodes_from([(x, self.database.nodes[x])
+                    #                        for x in self.database.nodes() if x not in differenceGraph.nodes()
+                    #                        ])
+                    # diffNew.add_edges_from([(x, y, self.database.edges[(x, y)])
+                    #                        for x, y in self.database.edges() if (x, y) not in differenceGraph.edges()
+                    #                        ])
+
+                    self.mainWindow.FCOM.syncDatabase(self.mainWindow.SETTINGS.value("Project/Server/Project"),
+                                                      differenceGraph)
+
+        self.dbLock.release()

Core/FrontendCommunicationsHandler.py

@@ -0,0 +1,779 @@
+#!/usr/bin/env python3
+
+
+from ast import literal_eval
+from typing import Union
+from pathlib import Path
+from msgpack import loads, dumps
+import socket
+import threading
+import time
+import re
+import networkx as nx
+
+from queue import Queue, Empty
+from base64 import b64encode, b64decode
+from PySide6 import QtCore
+from uuid import uuid4
+from hashlib import sha3_512
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.serialization import Encoding
+from cryptography.hazmat.primitives.serialization import PublicFormat
+from cryptography.hazmat.primitives.serialization import load_der_public_key
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.exceptions import InvalidTag
+
+# All functions use this to run. Checks if the program is closing.
+closeSoftwareLock = threading.Lock()
+closeSoftware = False
+
+
+# Has to be QObject for Signals to work.
+class CommunicationsHandler(QtCore.QObject):
+    """
+    This class handles communication with a LinkScope server.
+
+    Convention for variables changes here - all variables that have to do with network communications
+      are in snake_case. This makes them easy to differentiate from others, if it is important to track
+      what came from a network and what did not. This is also the case server-side.
+    """
+
+    connected_to_server_listener = QtCore.Signal(str)
+    receive_question_answer = QtCore.Signal(dict)
+    receive_chat_message = QtCore.Signal(str)
+    receive_resolutions_signal = QtCore.Signal(dict)
+    receive_completed_resolution_result_signal = QtCore.Signal(str, list)
+    receive_completed_resolution_string_result_signal = QtCore.Signal(str, str)
+    receive_document_summary_signal = QtCore.Signal(str, str)
+    remove_server_resolution_from_running_signal = QtCore.Signal(str)
+    receive_projects_list_signal = QtCore.Signal(list)
+    receive_project_canvases_list_signal = QtCore.Signal(list)
+    open_project_signal = QtCore.Signal(str)
+    close_project_signal = QtCore.Signal()
+    open_project_canvas_signal = QtCore.Signal(str)
+    close_project_canvas_signal = QtCore.Signal(str)
+    receive_project_database_update = QtCore.Signal(dict, bool)
+    receive_project_canvas_update_node = QtCore.Signal(str, str)
+    receive_project_canvas_update_link = QtCore.Signal(str, tuple)
+    receive_sync_database = QtCore.Signal(dict, dict)
+    status_message_signal = QtCore.Signal(str, bool)
+    receive_project_file_list = QtCore.Signal(list)
+    file_upload_finished_signal = QtCore.Signal(str)
+    file_upload_abort_signal = QtCore.Signal(str)
+    receive_sync_canvas_signal = QtCore.Signal(str, dict, dict)
+
+    def __init__(self, mainWindow):
+
+        super().__init__()
+
+        self.inbox = Queue()
+
+        self.mainWindow = mainWindow
+        self.downloadingFiles = {}
+        self.uploadingFiles = {}
+
+        self.connected_to_server_listener.connect(self.mainWindow.connectedToServerListener)
+        self.receive_question_answer.connect(self.mainWindow.questionAnswerListener)
+        self.receive_chat_message.connect(self.mainWindow.receiveChatMessage)
+        self.receive_resolutions_signal.connect(self.mainWindow.addResolutionsFromServerListener)
+        self.receive_completed_resolution_result_signal.connect(self.mainWindow.resolutionSignalListener)
+        self.receive_completed_resolution_string_result_signal.connect(self.mainWindow.resolutionSignalListener)
+        self.receive_document_summary_signal.connect(self.mainWindow.receiveSummaryOfDocument)
+        self.remove_server_resolution_from_running_signal.connect(self.mainWindow.cleanServerResolutionListener)
+        self.receive_projects_list_signal.connect(self.mainWindow.receiveProjectsListListener)
+        self.receive_project_canvases_list_signal.connect(self.mainWindow.receiveProjectCanvasesListListener)
+        self.open_project_signal.connect(self.mainWindow.openServerProjectListener)
+        self.open_project_canvas_signal.connect(self.mainWindow.openServerCanvasListener)
+        self.close_project_signal.connect(self.mainWindow.closeCurrentServerProject)
+        self.close_project_canvas_signal.connect(self.mainWindow.closeServerCanvasListener)
+        self.receive_project_database_update.connect(self.mainWindow.receiveServerDatabaseUpdate)
+        self.receive_project_canvas_update_node.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_project_canvas_update_link.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_sync_database.connect(self.mainWindow.receiveSyncDatabaseListener)
+        self.receive_project_file_list.connect(self.mainWindow.receiveFileListListener)
+        self.file_upload_finished_signal.connect(self.mainWindow.fileUploadFinishedListener)
+        self.file_upload_abort_signal.connect(self.mainWindow.receiveAbortUploadOfFiles)
+        self.receive_sync_canvas_signal.connect(self.mainWindow.receiveSyncCanvasListener)
+        self.status_message_signal.connect(self.mainWindow.statusMessageListener)
+
+        self.cipher = None
+        self.threadInc = None
+        self.threadInb = None
+        self.sock = None
+
+    def beginCommunications(self, password: str, server: str, port: int = 3777):
+        global closeSoftwareLock
+        global closeSoftware
+        with closeSoftwareLock:  # Not strictly necessary, but might as well just in case.
+            closeSoftware = False
+        try:
+            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+            self.sock.connect((server, port))
+
+            private_key = ec.generate_private_key(ec.SECP521R1())
+            self.sock.send(private_key.public_key().public_bytes(
+                Encoding.DER,
+                format=PublicFormat.SubjectPublicKeyInfo))
+            peer_public = load_der_public_key(self.sock.recv(5120))
+            shared_key = private_key.exchange(ec.ECDH(), peer_public)
+            derived_key = HKDF(
+                algorithm=hashes.SHA3_256(),
+                length=32,
+                salt=None,
+                info=b'Handshake'
+            ).derive(shared_key)
+            iv = sha3_512(derived_key).digest()[:16]
+            self.cipher = Cipher(algorithms.AES(derived_key), modes.CBC(iv))
+            encryptor = self.cipher.encryptor()
+            decrypter = self.cipher.decryptor()
+
+            passPad = 16 - (len(password) % 16)
+            passMessage = encryptor.update(password.encode() + b"*" * passPad) + encryptor.finalize()
+            self.sock.send(passMessage)
+            messageReceived = decrypter.update(self.sock.recv(5120)) + decrypter.finalize()
+            if messageReceived == b"Passphrase is OK":
+                self.threadInc = threading.Thread(target=self.scanIncoming)
+                self.threadInc.setDaemon(True)
+                self.threadInc.start()
+
+                self.threadInb = threading.Thread(target=self.scanInbox)
+                self.threadInb.setDaemon(True)
+                self.threadInb.start()
+
+                return True
+            elif messageReceived == b"Wrong Passphrase":
+                raise ValueError('Incorrect Passphrase')
+
+            self.mainWindow.MESSAGEHANDLER.warning("Server did not reply in the expected manner to "
+                                                   "password authentication.", popUp=True)
+        except Exception as exception:
+            self.mainWindow.MESSAGEHANDLER.error("Did not connect: " + str(exception))
+
+        try:
+            if self.sock is not None:
+                self.sock.shutdown(socket.SHUT_RDWR)
+                self.sock.close()
+                self.sock = None
+        except OSError:
+            # This would typically occur if the socket is already closed.
+            self.mainWindow.MESSAGEHANDLER.info('Tried to close socket that was already closed.')
+
+        return False
+
+    def isConnected(self):
+        """
+        Check if socket is in a working state.
+        """
+        if self.sock is not None and self.sock.fileno() != -1:
+            return True
+        return False
+
+    def close(self):
+        global closeSoftwareLock
+        global closeSoftware
+        with closeSoftwareLock:
+            closeSoftware = True
+        self.closeSocket()
+        for fileName in self.downloadingFiles:
+            # Close open file handlers
+            self.downloadingFiles[fileName].close()
+            # saveDir = Path(fileName)
+            saveDir = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / fileName
+            if saveDir.exists():
+                # Delete partly downloaded files.
+                saveDir.unlink(missing_ok=True)
+        self.downloadingFiles = {}
+        self.uploadingFiles = {}
+
+    def encryptTransmission(self, bytesObject):
+        encryptor = self.cipher.encryptor()
+        # No need to check for overflows or negative pads - transmitMessage practically ensures that the data
+        #   sent will always be less than or equal to 1280 bytes.
+        padNeeded = 1280 - len(bytesObject)
+        message = encryptor.update(b'a' * padNeeded + bytesObject) + encryptor.finalize()
+        return message
+
+    def decryptTransmission(self, bytesObject):
+        decrypter = self.cipher.decryptor()
+        try:
+            message = decrypter.update(bytesObject) + decrypter.finalize()
+            return message
+        except InvalidTag:
+            # If the ciphertext cannot be decrypted to a valid message, return None.
+            return None
+
+    def transmitMessage(self, messageJson: dict, showErrorOnBrokenPipe: bool = True):
+        # Note that Base64 encoded data is about 4/3 times the size of the original.
+        # 768 * 4/3 = 1024
+        print('Sending message:', messageJson)
+        argEncoded = b64encode(str(messageJson).encode())
+        largeMessageUUID = str(uuid4())
+        try:
+            for data in range(0, len(argEncoded), 768):
+                partArg = argEncoded[data:data + 768]
+                done = data + 768 >= len(argEncoded)
+                messageJson = {"uuid": largeMessageUUID,
+                               "message": partArg.decode(),
+                               "done": done}
+                self.sock.send(self.encryptTransmission(dumps(messageJson)))
+        except BrokenPipeError:
+            if showErrorOnBrokenPipe:
+                self.mainWindow.MESSAGEHANDLER.error("Disconnected from server!", popUp=True, exc_info=False)
+                self.mainWindow.disconnectFromServer()
+
+    def closeSocket(self):
+        """
+        Called by the close function in this class when communications with the server are to be ended.
+        :return:
+        """
+        try:
+            self.transmitMessage({"Operation": "Close Socket", "Arguments": {}}, showErrorOnBrokenPipe=False)
+        except OSError:
+            # Typically this is due to bad file descriptor, i.e. server is closed.
+            pass
+        except AttributeError:
+            # This happens if no connection was established while the software was running
+            pass
+        finally:
+            try:
+                if self.sock is not None:
+                    self.sock.shutdown(socket.SHUT_RDWR)
+                    self.sock.close()
+                    self.sock = None
+            except OSError:
+                # This would typically occur if the socket is already closed.
+                pass
+
+    def scanIncoming(self):
+        """
+        This function listens for incoming data, and puts it in the queue if
+        it exists.
+        """
+        preInbox = {}
+        while True:
+            try:
+                receivedInfo = self.sock.recv(5120)
+                if receivedInfo == b'':
+                    # Socket closed.
+                    break
+                for message in range(0, len(receivedInfo), 1280):
+                    decryptedInfo = self.decryptTransmission(receivedInfo[message:message + 1280])
+                    if decryptedInfo is None:
+                        self.mainWindow.MESSAGEHANDLER.warning("Invalid message received from server.")
+                        continue
+                    receivedMessage = loads(re.sub(b'^a*', b'', decryptedInfo, count=1))
+                    messageID = receivedMessage.get("uuid")
+                    if messageID in preInbox:
+                        preInbox[messageID]["message"] += receivedMessage.get("message")
+                    else:
+                        preInbox[messageID] = receivedMessage
+                    if receivedMessage.get("done"):
+                        preInbox[messageID]["message"] = literal_eval(
+                            b64decode(preInbox[messageID]["message"]).decode())
+                        self.inbox.put(preInbox.pop(messageID).get("message"))
+
+            except socket.error as socketError:
+                self.mainWindow.MESSAGEHANDLER.error('Socket Error: ' + str(socketError))
+                # If something happens, wait 2 seconds then try again.
+                closeSoftwareLock.acquire()
+                if not closeSoftware:
+                    closeSoftwareLock.release()
+                    time.sleep(2)
+                else:
+                    closeSoftwareLock.release()
+                    break
+            except ValueError:
+                # E.g.: Unpack failed: incomplete input
+                pass
+
+    def askServerForResolutions(self):
+        message = {"Operation": "Get Server Resolutions",
+                   "Arguments": {}}
+        self.transmitMessage(message)
+
+    def receiveResolutions(self, server_resolutions):
+        self.receive_resolutions_signal.emit(server_resolutions)
+
+    def runRemoteResolution(self, resolution_name: str, resolution_entities: list, resolution_parameters: dict,
+                            resolution_uid: str):
+        resolution_entities_to_send = []
+        for entity in resolution_entities:
+            try:
+                dereferenced_entity = dict(entity)
+                resolution_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any resolution as of now: 2022/1/2.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+            except KeyError:
+                pass
+        message = {'Operation': 'Run Resolution',
+                   'Arguments': {
+                       'resolution_name': resolution_name,
+                       'resolution_entities': resolution_entities_to_send,
+                       'resolution_parameters': resolution_parameters,
+                       'resolution_uid': resolution_uid
+                   }}
+        self.transmitMessage(message)
+
+    def receiveResolutionResult(self, resolution_name: str, resolution_result: Union[list, str], resolution_uid: str):
+        if isinstance(resolution_result, str):
+            self.receive_completed_resolution_string_result_signal.emit(resolution_name, resolution_result)
+        else:
+            self.receive_completed_resolution_result_signal.emit(resolution_name, resolution_result)
+        self.remove_server_resolution_from_running_signal.emit(resolution_uid)
+
+    def abortResolution(self, resolution_name: str, resolution_uid: str):
+        message = {'Operation': 'Abort Resolution',
+                   'Arguments': {
+                       'resolution_name': resolution_name,
+                       'resolution_uid': resolution_uid
+                   }}
+        self.transmitMessage(message)
+
+    def askProjectsList(self):
+        message = {'Operation': 'Get Projects List',
+                   'Arguments': {}}
+        self.transmitMessage(message)
+
+    def receiveProjectsList(self, projects: list):
+        self.receive_projects_list_signal.emit(projects)
+
+    def createProject(self, projectName: str, projectPassword: str):
+        if self.isConnected():
+            message = {'Operation': 'Create Project',
+                       'Arguments': {'project_name': projectName, 'password': projectPassword}}
+            self.transmitMessage(message)
+
+    def openProject(self, project_name: str, projectPassword: str):
+        if self.isConnected():
+            message = {'Operation': 'Open Project',
+                       'Arguments': {'project_name': project_name, 'password': projectPassword}}
+            self.transmitMessage(message)
+
+    def closeProject(self, project_name: str):
+        if self.isConnected():
+            message = {'Operation': 'Close Project',
+                       'Arguments': {'project_name': project_name}}
+            self.transmitMessage(message)
+
+    def askProjectCanvasesList(self, project_name: str):
+        if self.isConnected():
+            message = {'Operation': 'List Synced Canvases',
+                       'Arguments': {'project_name': project_name}}
+            self.transmitMessage(message)
+
+    def receiveProjectCanvasesList(self, canvases: list):
+        self.receive_project_canvases_list_signal.emit(canvases)
+
+    def askQuestion(self, project_name: str, question: str, reader_value: int, retriever_value: int, answer_count: int):
+        if self.isConnected():
+            message = {"Operation": "Ask Question",
+                       "Arguments": {
+                           'project_name': project_name,
+                           'question': question[:256],
+                           'reader_value': reader_value,
+                           'retriever_value': retriever_value,
+                           'answer_count': answer_count}}
+            self.transmitMessage(message)
+
+    def receiveQuestionAnswer(self, answer: list):
+        self.receive_question_answer.emit(answer)
+
+    def receiveTextMessage(self, chat_message: str):
+        self.receive_chat_message.emit(chat_message)
+
+    def sendTextMessage(self, project_name: str, chat_message: str):
+        if self.isConnected():
+            message = {"Operation": "Chat",
+                       "Arguments": {
+                           "project_name": project_name,
+                           "chat_message": chat_message[:1024]}}
+            self.transmitMessage(message)
+
+    def syncDatabase(self, project_name: str, client_project_graph: nx.DiGraph):
+        message = {'Operation': 'Sync Database',
+                   'Arguments': {
+                       'project_name': project_name,
+                       'client_project_graph': str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(
+                           client_project_graph))
+                   }}
+        self.transmitMessage(message)
+
+    def receiveSyncDatabase(self, database: str):
+        database_nodes, database_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(database)
+        self.receive_sync_database.emit(database_nodes, database_edges)
+
+    def askServerForFileList(self, project_name: str):
+        message = {"Operation": "Get File List",
+                   "Arguments": {'project_name': project_name}}
+        self.transmitMessage(message)
+
+    def syncCanvasSend(self, project_name: str, canvas_name: str, canvas_graph: nx.DiGraph):
+        if self.isConnected():
+            message = {'Operation': "Sync Canvas",
+                       'Arguments': {
+                           'project_name': project_name,
+                           'canvas_name': canvas_name,
+                           "canvas_graph": str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(canvas_graph))}}
+            self.transmitMessage(message)
+
+    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: str):
+        graph_nodes, graph_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(canvas_graph)
+        self.receive_sync_canvas_signal.emit(canvas_name, graph_nodes, graph_edges)
+
+    def closeCanvas(self, project_name: str, canvas_name: str):
+        if self.isConnected():
+            message = {'Operation': 'Close Canvas',
+                       'Arguments': {'project_name': project_name,
+                                     'canvas_name': canvas_name}}
+            self.transmitMessage(message)
+
+    # Items are sent both from the database and from the canvas.
+    # Items are added to the database from the canvas, but items can be
+    #   removed from the canvas without removing them from the database.
+    # Furthermore, one might want to add items to the the database, but not
+    #   to a particular canvas.
+    # Being verbose is better than prematurely optimizing for a few kbps of
+    #   network traffic.
+    def receiveDatabaseUpdateEvent(self, entity_json: dict, add: bool):
+        try:
+            entity_json['Icon'] = QtCore.QByteArray(b64decode(entity_json['Icon']))
+        except KeyError:
+            pass
+        self.receive_project_database_update.emit(entity_json, add)
+
+    def sendDatabaseUpdateEvent(self, project_name: str, entity_json: dict, add: bool):
+        try:
+            entity_json['Icon'] = entity_json['Icon'].toBase64().data()
+        except KeyError:
+            pass
+        message = {"Operation": "Update Project Entities",
+                   "Arguments": {
+                       'project_name': project_name,
+                       "entity_json": entity_json,
+                       "add": add}}
+        self.transmitMessage(message)
+
+    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
+        if isinstance(entity_or_link_uid, str):
+            self.receive_project_canvas_update_node.emit(canvas_name, entity_or_link_uid)
+        else:
+            self.receive_project_canvas_update_link.emit(canvas_name, entity_or_link_uid)
+
+    def sendCanvasUpdateEvent(self, project_name: str, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
+        message = {"Operation": "Update Canvas Entities",
+                   "Arguments": {
+                       'project_name': project_name,
+                       "canvas_name": canvas_name,
+                       "entity_or_link_uid": entity_or_link_uid}}
+        self.transmitMessage(message)
+
+    def receiveFileList(self, file_list: list):
+        self.receive_project_file_list.emit(file_list)
+
+    def sendFile(self, project_name: str, file_name: str, filePath: Path):
+        """
+        Starts a thread, calling sendFileHelper to send the file specified.
+
+        :param project_name:
+        :param file_name:
+        :param filePath:
+        :return:
+        """
+        sendHelperThread = threading.Thread(target=self.sendFileHelper, args=(project_name, file_name, filePath))
+        sendHelperThread.setDaemon(True)
+        self.uploadingFiles[file_name] = sendHelperThread
+        sendHelperThread.start()
+
+    def sendFileHelper(self, project_name: str, file_name: str, filePath: Path):
+        """
+        Sends file in chunks to avoid loading the entire thing in memory.
+
+        :param project_name:
+        :param file_name:
+        :param filePath:
+        :return:
+        """
+        if not filePath.exists() or not filePath.is_file():
+            return
+        fileHandler = open(filePath, 'rb')
+        print('Sending file:', filePath)
+
+        currThread = threading.currentThread()
+        while getattr(currThread, "continue_running", True):
+            filePart = fileHandler.read(512)
+            if not filePart:
+                messageJson = {"Operation": "File Upload Done",
+                               "Arguments": {
+                                   'project_name': project_name,
+                                   'file_name': file_name
+                               }}
+                self.transmitMessage(messageJson)
+                break
+            messageJson = {"Operation": "File Upload",
+                           "Arguments": {
+                               'project_name': project_name,
+                               'file_name': file_name,
+                               'file_contents': filePart
+                           }}
+            self.transmitMessage(messageJson)
+        fileHandler.close()
+
+    def sendFileAbort(self, project_name: str, file_name: str):
+        try:
+            uploadToAbort = self.uploadingFiles.pop(file_name)
+            uploadToAbort.continue_running = False
+            messageJson = {"Operation": "File Upload Abort",
+                           "Arguments": {
+                               'project_name': project_name,
+                               'file_name': file_name
+                           }}
+            self.transmitMessage(messageJson)
+        except KeyError:
+            pass
+
+    def scanInbox(self):
+        """
+        This function checks if there is anything in the inbox, and if
+        there is, calls the appropriate functions.
+        """
+        prevMesg = None
+        while True:
+            try:
+                message = self.inbox.get(timeout=0.2)
+            except Empty:
+                with closeSoftwareLock:
+                    if not closeSoftware:
+                        time.sleep(0.2)
+                        continue
+                    else:
+                        return
+            if prevMesg == message:
+                # Same message, do not waste time handling.
+                continue
+            print('Message To handle:', message)  # TODO Make this logging.
+            operation = message['Operation']
+            arguments = message['Arguments']
+            if operation == 'Get Server Resolutions':
+                self.receiveResolutions(**arguments)
+            elif operation == 'Get Projects List':
+                self.receiveProjectsList(**arguments)
+            elif operation == 'Status Message':
+                self.handleStatusMessage(**arguments)
+            elif operation == 'List Synced Canvases':
+                self.receiveProjectCanvasesList(**arguments)
+            elif operation == 'Resolution Result':
+                self.receiveResolutionResult(**arguments)
+            elif operation == "Chat":
+                self.receiveTextMessage(**arguments)
+            elif operation == "Sync Database":
+                self.receiveSyncDatabase(**arguments)
+            elif operation == "Sync Canvas":
+                self.receiveSyncCanvas(**arguments)
+            elif operation == "Answer Question":
+                self.receiveQuestionAnswer(**arguments)
+            elif operation == "Update Project Entities":
+                self.receiveDatabaseUpdateEvent(**arguments)
+            elif operation == "Update Canvas Entities":
+                self.receiveCanvasUpdateEvent(**arguments)
+            elif operation == "File List":
+                self.receiveFileList(**arguments)
+            elif operation == "File Download":
+                self.receiveFileListener(**arguments)
+            elif operation == "Get File Summary":
+                self.receiveFileSummaryListener(**arguments)
+            elif operation == "File Upload Abort":
+                self.receiveFileUploadAbort(**arguments)
+            elif operation == "Delete File":
+                pass
+            else:
+                self.mainWindow.MESSAGEHANDLER.warning('Unhandled message: ' + str(message) +
+                                                       ' On Operation: ' + str(operation))
+            prevMesg = message
+
+    def handleStatusMessage(self, operation: str, message: str, status_code: int):
+        """
+        Operations that are completely server-side, or do not conform to the query - response model,
+        send status messages to inform the client of what is going on.
+
+        Status codes follow the same pattern as the ones in HTTP:
+        200 - Operation Successful.
+        404 - A requested resource was not found
+        403 - Access to the requested resource is forbidden
+        500 - Server could not adequately handle the request.
+        :param operation:
+        :param message:
+        :param status_code:
+        :return:
+        """
+        if status_code != 200:
+            self.status_message_signal.emit('Operation ' + operation + ' failed with status code ' +
+                                            str(status_code) + ': ' + message, True)
+        else:
+            if operation == 'Create Project':
+                # No need to do anything here. Creating a new project also opens it.
+                pass
+            elif operation == 'Open Project':
+                projectName = message.split(': ', 1)[1]
+                self.open_project_signal.emit(projectName)
+            # Show the user that the server is doing something.
+            elif operation == 'Opening Project':
+                self.status_message_signal.emit(message, True)
+            elif operation == 'Close Project':
+                self.close_project_signal.emit()
+            elif operation == 'Create Canvas':
+                # No need to do anything here. Creating a new canvas also opens it.
+                pass
+            elif operation == 'Open Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.open_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Close Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.close_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Connect To Server':
+                server_name = message.split(': ', 1)[1]
+                self.connected_to_server_listener.emit(server_name)
+            elif operation == 'File Upload':
+                file_name = message.split(': ', 1)[1]
+                self.file_upload_finished_signal.emit(file_name)
+            elif operation == 'File Download Done':
+                file_name = message.split(': ', 1)[1]
+                self.receiveFileDoneListener(file_name)
+            elif operation == 'Abort Resolution':
+                # Remove resolution from resolutions list.
+                resolution_uid = message.split(': ', 1)[1]
+                self.remove_server_resolution_from_running_signal.emit(resolution_uid)
+            elif operation == 'Delete File':
+                # Remove file from uploaded files list.
+                pass  # TODO
+            elif operation == 'File Upload Abort':
+                file_name = message.split(': ', 1)[1]
+                # Remove file from uploading files list.
+                self.file_upload_abort_signal.emit(file_name)
+            else:
+                self.mainWindow.MESSAGEHANDLER.warning('Unhandled status message: ' + message +
+                                                       ' Code: ' + str(status_code) +
+                                                       ' On Operation: ' + str(operation))
+
+    def receiveFile(self, project_name: str, file_name: str):
+        # Do not download files already being downloaded.
+        if self.downloadingFiles.get(file_name) is None:
+            saveDir = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
+            fileHandler = open(saveDir, "wb")
+            self.downloadingFiles[file_name] = fileHandler
+
+            message = {'Operation': 'Download File',
+                       'Arguments': {
+                           'project_name': project_name,
+                           'file_name': file_name
+                       }}
+            self.transmitMessage(message)
+
+    def receiveFileListener(self, file_name: str, file_contents: bytes):
+        try:
+            fileHandler = self.downloadingFiles.get(file_name)
+            fileHandler.write(file_contents)
+        except Exception:
+            # In case the file is deleted in the middle of writing, or anything else going wrong.
+            self.mainWindow.MESSAGEHANDLER.warning('Received data for file: ' + file_name +
+                                                   ' but no valid file handler exists for this file.')
+
+    def receiveFileDoneListener(self, file_name: str):
+        fileHandler = self.downloadingFiles.pop(file_name)
+        if fileHandler is None:
+            self.mainWindow.MESSAGEHANDLER.warning('Received file: ' + file_name +
+                                                   ' but no file handler exists for this file.')
+            return
+
+        fileHandler.close()
+        self.status_message_signal.emit('Finished downloading file from server: ' + file_name, False)
+
+    def receiveFileAbort(self, project_name: str, file_name: str):
+        messageJson = {"Operation": "Download File Abort",
+                       "Arguments": {
+                           'project_name': project_name,
+                           'file_name': file_name
+                       }}
+        self.transmitMessage(messageJson)
+        fileHandler = self.downloadingFiles.pop(file_name)
+        if fileHandler is None:
+            return
+        fileHandler.close()
+        abortedPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
+        abortedPath.unlink(missing_ok=True)
+
+    def deleteFile(self, project_name: str, file_name: str):
+        message = {"Operation": "Delete File",
+                   "Arguments": {
+                       'project_name': project_name,
+                       'file_name': file_name}}
+        self.transmitMessage(message)
+
+    def askServerForFileSummary(self, project_name: str, document_name: str):
+        message = {"Operation": "Get File Summary",
+                   "Arguments": {
+                       'project_name': project_name,
+                       'document_name': document_name}}
+        self.transmitMessage(message)
+
+    def receiveFileSummaryListener(self, document_name: str, summary: str):
+        self.receive_document_summary_signal.emit(document_name, summary)
+
+    def receiveFileUploadAbort(self, file_name: str):
+        """
+        If we are told by the server to stop uploading a file,
+        we should do so (i.e. because no space left on server),
+        to avoid wasting bandwidth.
+
+        :param file_name:
+        :return:
+        """
+        try:
+            uploadToAbort = self.uploadingFiles.pop(file_name)
+            uploadToAbort.continue_running = False
+            self.file_upload_abort_signal.emit(file_name)
+        except KeyError:
+            pass
+
+    def sendFileAbortAll(self, project_name: str):
+        """
+        Abort the sending of all files currently being transmitted.
+
+        :param project_name:
+        :return:
+        """
+        for file_name in self.uploadingFiles:
+            try:
+                uploadToAbort = self.uploadingFiles.pop(file_name)
+                uploadToAbort.continue_running = False
+                messageJson = {"Operation": "File Upload Abort",
+                               "Arguments": {
+                                   'project_name': project_name,
+                                   'file_name': file_name
+                               }}
+                self.transmitMessage(messageJson)
+            except KeyError:
+                pass
+
+    def receiveFileAbortAll(self, project_name: str):
+        """
+        Abort the downloading of all files currently being transmitted.
+
+        :param project_name:
+        :return:
+        """
+        for file_name in self.downloadingFiles:
+            messageJson = {"Operation": "Download File Abort",
+                           "Arguments": {
+                               'project_name': project_name,
+                               'file_name': file_name
+                           }}
+            self.transmitMessage(messageJson)
+            fileHandler = self.downloadingFiles.pop(file_name)
+            if fileHandler is None:
+                return
+            fileHandler.close()
+            abortedPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
+            abortedPath.unlink(missing_ok=True)

Core/Interface/DockBarOne.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python3
+from json import dumps
+from Core.Interface.Entity import BaseNode
+from Core.Interface import Stylesheets
+from PySide6 import QtWidgets, QtCore, QtGui
+
+
+class DockBarOne(QtWidgets.QDockWidget):
+
+    def initialiseLayout(self):
+        childWidget = QtWidgets.QTabWidget()
+        nodeTabChildWidget = QtWidgets.QWidget()
+        docsTabChildWidget = QtWidgets.QWidget()
+        nodeTabChildWidget.setLayout(QtWidgets.QGridLayout())
+        docsTabChildWidget.setLayout(QtWidgets.QGridLayout())
+        self.setWidget(childWidget)
+
+        childWidget.addTab(nodeTabChildWidget, "Node Operations")
+        childWidget.addTab(docsTabChildWidget, "Documents")
+
+        nodeTabChildWidget.layout().addWidget(self.nodesPalette)
+        nodeTabChildWidget.layout().addWidget(self.resolutionsPalette)
+        docsTabChildWidget.layout().addWidget(self.documentsList)
+        docsTabChildWidget.layout().addWidget(self.existingEntitiesPalette)
+
+        worldDocToggle = ToggleWorldDocButton(self, self.parent())
+        docsTabChildWidget.layout().addWidget(worldDocToggle)
+
+    def __init__(self,
+                 mainWindow,
+                 resolutionManager,
+                 resourceHandler,
+                 entityDatabase,
+                 title="DockBar One"):
+        super(DockBarOne, self).__init__(parent=mainWindow)
+        self.resolutionManager = resolutionManager
+        self.resourceHandler = resourceHandler
+        self.lentDB = entityDatabase
+        self.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
+                             QtCore.Qt.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.setObjectName(title)
+
+        self.nodesPalette = NodeList(self.resourceHandler, self)
+        self.resolutionsPalette = ResolutionList(self.resolutionManager,
+                                                 self.lentDB,
+                                                 self.parent(),
+                                                 self)
+        self.documentsList = DocList(self.resourceHandler, self)
+        self.existingEntitiesPalette = EntityList(self.lentDB, self.parent(), self)
+
+        self.initialiseLayout()
+
+
+class ToggleWorldDocButton(QtWidgets.QPushButton):
+
+    def __init__(self, parent=None, mainWindow=None):
+        super(ToggleWorldDocButton, self).__init__(parent=parent)
+        self.mainWindow = mainWindow
+        self.setText("Toggle Detailed View")
+        self.clicked.connect(self.buttonPressed)
+
+    def buttonPressed(self):
+        if self.mainWindow is not None:
+            self.mainWindow.toggleWorldDoc()
+
+
+class EntityList(QtWidgets.QTreeWidget):
+
+    def __init__(self, entityDB, mainWindow, parent=None):
+        super(EntityList, self).__init__(parent=parent)
+
+        self.entityDB = entityDB
+        self.mainWindow = mainWindow
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Entity List'])
+        self.setAlternatingRowColors(False)
+        self.setMinimumWidth(200)
+        self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
+        self.menu = QtWidgets.QMenu()
+
+        actionDelete = QtGui.QAction('Delete Selected Items',
+                                     self.menu,
+                                     statusTip="Delete the selected entities from the database.",
+                                     triggered=self.deleteSelectedItems)
+        self.menu.addAction(actionDelete)
+        self.menu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
+
+        self.entityCategories: dict = {}
+        self.entityTypes: dict = {}
+        self.loadEntities()
+
+    def loadEntities(self):
+        self.clear()
+        self.entityCategories = {}
+        self.entityTypes = {}
+
+        for category in self.mainWindow.RESOURCEHANDLER.getEntityCategories():
+            catTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            self.entityCategories[category] = catTreeItem
+
+        for category in self.entityCategories:
+            for entityType in self.mainWindow.RESOURCEHANDLER.getAllEntitiesInCategory(category):
+                typeTreeItem = QtWidgets.QTreeWidgetItem(self.entityCategories[category], [entityType])
+                self.entityTypes[entityType] = typeTreeItem
+
+        allEntities = self.entityDB.getAllEntities()
+        for entity in allEntities:
+            topItem = self.entityTypes[entity['Entity Type']]
+            primaryAttr = entity[list(entity)[1]]
+            pixmapIcon = QtGui.QPixmap()
+            pixmapIcon.loadFromData(entity['Icon'])
+            EntityWidget(topItem,
+                         entity['uid'],
+                         QtGui.QIcon(pixmapIcon),
+                         primaryAttr)
+
+        # Hide entity types and categories with no item instances (children):
+        for category in self.entityCategories:
+            childrenHidden = 0
+            for entityTypeIndex in range(self.entityCategories[category].childCount()):
+                entityTypeItem = self.entityCategories[category].child(entityTypeIndex)
+                if entityTypeItem.childCount() == 0:
+                    entityTypeItem.setHidden(True)
+                    childrenHidden += 1
+            if childrenHidden == self.entityCategories[category].childCount():
+                self.entityCategories[category].setHidden(True)
+
+    def addEntity(self, entityJson):
+        primaryAttr = entityJson[list(entityJson)[1]]
+        entityTypeItem = self.entityTypes[entityJson['Entity Type']]
+        for entityNo in range(entityTypeItem.childCount()):
+            child = entityTypeItem.child(entityNo)
+            if child.uid == entityJson['uid']:
+                child.setText(0, primaryAttr)
+                return
+        pixmapIcon = QtGui.QPixmap()
+        pixmapIcon.loadFromData(entityJson['Icon'])
+        EntityWidget(entityTypeItem,
+                     entityJson['uid'],
+                     QtGui.QIcon(pixmapIcon),
+                     primaryAttr)
+        # Un-hide parents of item, if they were hidden.
+        if entityTypeItem.isHidden():
+            entityTypeItem.setHidden(False)
+            entityTypeItem.parent().setHidden(False)
+
+    def removeEntity(self, entityJson):
+        entityType = self.entityTypes[entityJson['Entity Type']]
+        for entityNo in range(entityType.childCount()):
+            child = entityType.child(entityNo)
+            if child.uid == entityJson['uid']:
+                entityType.removeChild(child)
+                del child
+                if entityType.childCount() == 0:
+                    entityType.setHidden(True)
+                    entityCategory = entityType.parent()
+                    entityCategory.setHidden(True)
+                    for childID in range(entityCategory.childCount()):
+                        child = entityCategory.child(childID)
+                        if not child.isHidden():
+                            entityCategory.setHidden(False)
+                            break
+                break
+
+    def mouseMoveEvent(self, event):
+        """
+        Handle dragging of entities onto canvas.
+        """
+        # No, I have no idea why this is the case: v
+        if event.button() == QtGui.Qt.MouseButton.NoButton:
+            itemDragged = self.itemAt(event.pos())
+
+            if itemDragged is None or \
+                    itemDragged.text(0) in self.entityTypes.keys() or \
+                    itemDragged.text(0) in self.entityCategories.keys():
+                return
+
+            self.setCurrentItem(itemDragged)
+            drag = QtGui.QDrag(self)
+            mimeData = QtCore.QMimeData()
+
+            mimeData.setText(dumps({'uid': itemDragged.uid}))
+            drag.setMimeData(mimeData)
+
+            # All Entities should have icons, but you never know.
+            pixmap = None
+            if itemDragged.icon(0) is not None:
+                pixmap = itemDragged.icon(0).pixmap(40, 40)
+                drag.setPixmap(pixmap)
+            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
+            drag.exec_()
+        else:
+            super().mouseMoveEvent(event)
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        super(EntityList, self).mousePressEvent(event)
+
+        if event.button() == QtGui.Qt.MouseButton.RightButton:
+            itemDragged = self.itemAt(event.pos())
+            if isinstance(itemDragged, EntityWidget):
+                self.menu.exec(QtGui.QCursor.pos())
+
+    def deleteSelectedItems(self):
+        itemsToDel = [item.uid for item in self.selectedItems() if isinstance(item, EntityWidget)]
+        for itemUID in itemsToDel:
+            self.mainWindow.deleteSpecificEntity(itemUID)
+
+
+class EntityWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, uid, icon=None, text=""):
+        super(EntityWidget, self).__init__(parent, [text])
+        self.uid = uid
+        if icon is not None:
+            self.setIcon(0, icon)
+
+
+class DocList(QtWidgets.QTreeWidget):
+
+    def __init__(self, resourceHandler, parent=None):
+        super(DocList, self).__init__(parent=parent)
+        self.resourceHandler = resourceHandler
+        self.setAlternatingRowColors(False)
+        self.setHeaderLabels(['Files Loaded'])
+        self.uploadingFileWidgets = []
+        self.uploadedFileWidgets = []
+        self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
+        self.menu = QtWidgets.QMenu()
+
+        actionDelete = QtGui.QAction('Delete Selected Files',
+                                     self.menu,
+                                     statusTip="Delete the selected files from the server.",
+                                     triggered=self.deleteSelectedFiles)
+        self.menu.addAction(actionDelete)
+        self.menu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
+
+    def addUploadingFileToList(self, fileName: str):
+        newWidget = DocWidget(self,
+                              QtGui.QIcon(self.resourceHandler.getIcon("uploading")), fileName)
+        self.uploadingFileWidgets.append(newWidget)
+
+    def finishUploadingFile(self, fileName: str):
+        # Remove uploading file. Server will send updated file list.
+        for doc in self.uploadingFileWidgets:
+            if doc.getFileName() == fileName:
+                self.takeTopLevelItem(self.indexOfTopLevelItem(doc))
+                self.uploadingFileWidgets.remove(doc)
+                break
+
+    def updateFileListFromServer(self, fileList):
+        # If disconnected from server, or project has an empty file list, clear the widget.
+        if not fileList:
+            self.clear()
+            self.uploadingFileWidgets = []
+            self.uploadedFileWidgets = []
+            return
+
+        for fileName in fileList:
+            if fileName in [uploadedFileWidget.getFileName() for uploadedFileWidget in self.uploadedFileWidgets]:
+                continue
+            newWidget = DocWidget(self, None, fileName)
+            self.uploadedFileWidgets.append(newWidget)
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        super(DocList, self).mousePressEvent(event)
+        # TODO - be able to drag downloaded files onto canvas, and create entities when that happens.
+
+    def deleteSelectedFiles(self):
+        filesToDel = [file.getFileName() for file in self.selectedItems() if isinstance(file, DocWidget)]
+        for fileName in filesToDel:
+            # TODO
+            pass
+            # self.mainWindow.deleteSpecificEntity(itemUID)
+
+
+class DocWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, text=""):
+        super(DocWidget, self).__init__(parent, [text])
+        if icon is not None:
+            self.setIcon(0, icon)
+
+    def getFileName(self):
+        return self.text(0)
+
+
+class ResolutionList(QtWidgets.QTreeWidget):
+
+    def __init__(self,
+                 resolutionManager,
+                 entityDatabase,
+                 mainWindow,
+                 parent=None):
+
+        super(ResolutionList, self).__init__(parent=parent)
+        self.resolutionManager = resolutionManager
+        self.lentDB = entityDatabase
+        self.mainWindow = mainWindow
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Resolutions'])
+        self.setAlternatingRowColors(False)
+        self.setMinimumWidth(200)
+
+        self.loadAllResolutions()
+
+    def loadAllResolutions(self):
+        self.clear()
+        for category in self.resolutionManager.getResolutionCategories():
+            resTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            resolutions = self.resolutionManager.getResolutionsInCategory(category)
+            for res in resolutions:
+                ResolutionWidget(resTreeItem, text=res)
+
+    def loadResolutionsForSelected(self, selected):
+        self.clear()
+        if len(selected) == 0:
+            self.loadAllResolutions()
+            return
+        entityTypes = set()
+        for item in selected:
+            if not isinstance(item, BaseNode):
+                continue
+            ent = self.lentDB.getEntity(item.uid)
+            entityTypes.add(ent['Entity Type'])
+        res = self.resolutionManager.getResolutionsForEntityTypesByCategory(entityTypes)
+        for category in res:
+            resolutions = res[category]
+            if len(resolutions) != 0:
+                resTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+                for resolution in resolutions:
+                    ResolutionWidget(resTreeItem, text=resolution)
+
+    def mouseDoubleClickEvent(self, event):
+        super(ResolutionList, self).mouseDoubleClickEvent(event)
+        resItem = self.itemAt(event.pos())
+        if resItem is None or not isinstance(resItem, ResolutionWidget):
+            return
+        # Resolution Widgets should always have a parent, no need to check.
+        category = resItem.parent().text(0)
+        resolution = resItem.text(0)
+        if resolution not in self.resolutionManager.getResolutionsInCategory(category):
+            return
+
+        self.mainWindow.runResolution(category + '/' + resolution)
+
+
+class ResolutionWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, text=""):
+        super(ResolutionWidget, self).__init__(parent, [text])
+        if icon is not None:
+            self.setIcon(0, icon)
+
+
+class NodeList(QtWidgets.QTreeWidget):
+
+    def __init__(self, resourceHandler, parent=None):
+        super(NodeList, self).__init__(parent=parent)
+        self.resourceHandler = resourceHandler
+        self.setDragEnabled(True)
+        self.setHeaderLabels(['Entities'])
+        self.setAlternatingRowColors(False)
+        self.allEntities = []
+
+        self.loadEntities()
+
+    def loadEntities(self):
+        self.clear()
+        self.allEntities = []
+        for category in self.resourceHandler.getEntityCategories():
+            catTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
+            entities = self.resourceHandler.getAllEntityDetailsWithIconsInCategory(
+                category)
+            for entity in entities:
+                NodeWidget(catTreeItem,
+                           QtGui.QIcon(entity[1]),
+                           entity[0]['Entity Type'],
+                           dumps(entity[0]))
+
+        self.allEntities = self.resourceHandler.getAllEntities()
+
+    def mouseMoveEvent(self, event):
+        """
+        Handle dragging of entities onto canvas.
+        """
+        # No, I have no idea why this is the case: v
+        if event.button() == QtGui.Qt.MouseButton.NoButton:
+            itemDragged = self.itemAt(event.pos())
+            if itemDragged is None or \
+                    itemDragged.text(0) not in self.allEntities:
+                return
+
+            self.setCurrentItem(itemDragged)
+            drag = QtGui.QDrag(self)
+            mimeData = QtCore.QMimeData()
+
+            mimeData.setText(itemDragged.entityJsonText)
+            drag.setMimeData(mimeData)
+
+            # All Entities should have icons, but you never know.
+            pixmap = None
+            if itemDragged.icon(0) is not None:
+                pixmap = itemDragged.icon(0).pixmap(40, 40)
+                drag.setPixmap(pixmap)
+            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
+            drag.exec_()
+        else:
+            # This should never happen.
+            super(NodeList, self).mousePressEvent(event)
+
+
+class NodeWidget(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, parent, icon=None, name="", entityJsonText=""):
+        super(NodeWidget, self).__init__(parent, [name])
+        if icon is not None:
+            self.setIcon(0, icon)
+        self.entityJsonText = entityJsonText

Core/Interface/DockBarThree.py

@@ -0,0 +1,546 @@
+#!/usr/bin/env python3
+
+from PySide6 import QtWidgets, QtCore, QtCharts, QtGui
+from Core.Interface import Stylesheets
+from datetime import datetime
+from getpass import getuser
+from time import sleep
+import networkx as nx
+import queue
+
+
+class DockBarThree(QtWidgets.QDockWidget):
+    """
+    Dockbar that hosts the Timeline, Chat and Server Status widgets.
+    """
+
+    def initialiseLayout(self):
+        # self.setStyleSheet(Stylesheets.MAIN_WINDOW_STYLESHEET)
+        childWidget = QtWidgets.QWidget()
+        childWidget.setLayout(QtWidgets.QVBoxLayout())
+        childWidget.setContentsMargins(0, 0, 0, 0)
+        self.setWidget(childWidget)
+
+        childWidget2 = QtWidgets.QWidget()
+        childWidget2.setContentsMargins(0, 0, 0, 0)
+        childWidget2.setLayout(QtWidgets.QHBoxLayout())
+        childWidget.layout().addWidget(childWidget2)
+        childWidget2.layout().addWidget(self.tabPane)
+        self.tabPane.setContentsMargins(0, 0, 0, 0)
+        self.tabPane.addTab(self.logViewer, 'Program Log')
+        self.tabPane.addTab(self.timeWidget, 'Timeline')
+        childWidget2.layout().addWidget(self.chatBox)
+        self.serverStatus.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
+        childWidget.layout().addWidget(self.serverStatus)
+
+    def __init__(self, mainWindow, title="Dockbar Three"):
+        super(DockBarThree, self).__init__(parent=mainWindow)
+        self.setAllowedAreas(QtCore.Qt.TopDockWidgetArea |
+                             QtCore.Qt.BottomDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.setObjectName(title)
+        self.setMaximumHeight(275)
+        self.setMinimumHeight(275)
+
+        self.tabPane = QtWidgets.QTabWidget()
+
+        self.serverStatus = ServerStatusBox(self)
+        self.chatBox = ChatBox(self, self.parent())
+        self.timeWidget = TimeWidget(self, self.parent())
+        self.logViewer = QtWidgets.QPlainTextEdit()
+
+        # Because we're not going to stop the thread before closing, an error will be thrown by Qt.
+        # That error can be safely ignored.
+        self.logViewerUpdateThread = LoggingUpdateThread(mainWindow.MESSAGEHANDLER)
+        self.logViewerUpdateThread.loggingSignal.connect(self.updateLogs)
+        self.logViewerUpdateThread.start()
+        self.logViewer.setReadOnly(True)
+
+        self.initialiseLayout()
+
+    def updateLogs(self, newLogMessage: str):
+        self.logViewer.appendPlainText(newLogMessage)
+
+
+class TimeWidget(QtWidgets.QWidget):
+
+    def __init__(self, parent, mainWindow):
+        super(TimeWidget, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.timeDetails = {}
+        self.currentTimeStep = []
+
+        self.timelineChart = QtCharts.QChart()
+        self.timelineChart.setTitle("Timeline")
+        self.timelineChart.setTheme(QtCharts.QChart.ChartThemeBlueCerulean)
+        self.timelineChart.setMargins(QtCore.QMargins(0, 0, 0, 0))
+        self.chartView = QtCharts.QChartView(self.timelineChart)
+        self.chartView.setRubberBand(QtCharts.QChartView.NoRubberBand)
+        self.timelineChart.setAnimationOptions(QtCharts.QChart.AllAnimations)
+        self.timelineChart.setAnimationDuration(250)
+        self.timelineChart.legend().hide()
+
+        self.timescaleSelector = TimelineTimescaleSelector(self)
+
+        self.setLayout(QtWidgets.QHBoxLayout())
+        self.layout().addWidget(self.timescaleSelector)
+        self.layout().addWidget(self.chartView)
+
+    def takePictureOfView(self, transparentBackground: bool = False):
+        # Need to set size and format of pic before using it.
+        # Ref: https://qtcentre.org/threads/10975-Help-Export-QGraphicsView-to-Image-File
+        # Rendering best optimized to rgb32 and argb32_premultiplied.
+        # Ref: https://doc.qt.io/qtforpython/PySide6/QtGui/QImage.html?highlight=qimage#image-formats
+        picture = QtGui.QImage(self.chartView.size(), QtGui.QImage.Format_ARGB32_Premultiplied)
+        picturePainter = QtGui.QPainter(picture)
+        if not transparentBackground:
+            picture.fill(QtGui.QColor(61, 61, 61))
+        self.chartView.render(picturePainter)
+        return picture
+
+    def updateTimeline(self, node, added: bool = True, updateGraph: bool = True):
+        nodeTime = datetime.fromisoformat(node['Date Created'])
+
+        nodeYear = nodeTime.year
+        nodeMonth = nodeTime.month
+        nodeDay = nodeTime.day
+        nodeHour = nodeTime.hour
+        nodeMinute = nodeTime.minute
+
+        # Dicts are fast, but this is not very efficient.
+        if nodeYear not in self.timeDetails:
+            self.timeDetails[nodeYear] = {}
+
+        if nodeMonth not in self.timeDetails[nodeYear]:
+            self.timeDetails[nodeYear][nodeMonth] = {}
+
+        if nodeDay not in self.timeDetails[nodeYear][nodeMonth]:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay] = {}
+
+        if nodeHour not in self.timeDetails[nodeYear][nodeMonth][nodeDay]:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour] = {}
+
+        if nodeMinute not in self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour]:
+            # Sanity check. Should not be able to remove nodes that do not exist, but you never know.
+            if added:
+                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 1
+            else:
+                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 0
+        else:
+            if added:
+                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] += 1
+            else:
+                if self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] > 0:
+                    self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] -= 1
+
+        if updateGraph:
+            self.drawChart([])
+
+    def resetTimeline(self, graph: nx.DiGraph, updateGraph: bool = True):
+        self.timeDetails = {}
+
+        for node in graph.nodes():
+            self.updateTimeline(graph.nodes[node], True, False)
+
+        if updateGraph:
+            self.drawChart([])
+
+    def drawChart(self, timescale: list):
+        try:
+            year = timescale[0]
+        except IndexError:
+            year = None
+        try:
+            month = timescale[1]
+        except IndexError:
+            month = None
+        try:
+            day = timescale[2]
+        except IndexError:
+            day = None
+        try:
+            hour = timescale[3]
+        except IndexError:
+            hour = None
+        try:
+            minute = timescale[4]
+        except IndexError:
+            minute = None
+
+        if minute is not None:
+            barsDict = {minute: self.timeDetails[year][month][day][hour][minute]}
+            self.currentTimeStep = [year, month, day, hour, minute]
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        elif hour is not None:
+            barsDict = {}
+            for minute in self.timeDetails[year][month][day][hour]:
+                barsDict[minute] = self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year, month, day, hour]
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        elif day is not None:
+            barsDict = {}
+            for hour in self.timeDetails[year][month][day]:
+                barsDict[hour] = 0
+                for minute in self.timeDetails[year][month][day][hour]:
+                    barsDict[hour] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year, month, day]
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        elif month is not None:
+            barsDict = {}
+            for day in self.timeDetails[year][month]:
+                barsDict[day] = 0
+                for hour in self.timeDetails[year][month][day]:
+                    for minute in self.timeDetails[year][month][day][hour]:
+                        barsDict[day] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year, month]
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        elif year is not None:
+            barsDict = {}
+            for month in self.timeDetails[year]:
+                barsDict[month] = 0
+                for day in self.timeDetails[year][month]:
+                    for hour in self.timeDetails[year][month][day]:
+                        for minute in self.timeDetails[year][month][day][hour]:
+                            barsDict[month] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = [year]
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        else:
+            barsDict = {}
+            for year in self.timeDetails:
+                barsDict[year] = 0
+                for month in self.timeDetails[year]:
+                    for day in self.timeDetails[year][month]:
+                        for hour in self.timeDetails[year][month][day]:
+                            for minute in self.timeDetails[year][month][day][hour]:
+                                barsDict[year] += self.timeDetails[year][month][day][hour][minute]
+            self.currentTimeStep = []
+            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+    def drawChartHelper(self, barsDict: dict, timestep: list):
+        timelineSeries = QtCharts.QBarSeries(self.timelineChart)
+        maxEntityNum = 0
+        xAxisValues = []
+
+        barSet = TimelineBarSet('Entities', self, timestep, list(barsDict))
+        barSet.setColor(QtGui.Qt.darkCyan)
+        for bar in barsDict:
+            barSet.append(barsDict[bar])
+            timelineSeries.append(barSet)
+            if barsDict[bar] > maxEntityNum:
+                maxEntityNum = barsDict[bar]
+            value = ""
+            for step in range(len(timestep)):
+                if step <= 2:
+                    value += str(timestep[step]) + '/'
+                    if step == 2:
+                        value = value[:-1]
+                        value += " "
+                else:
+                    value += str(timestep[step]) + ':'
+            value += str(bar)
+            xAxisValues.append(value)
+        self.timelineChart.removeAllSeries()
+        self.timelineChart.addSeries(timelineSeries)
+        self.timelineChart.createDefaultAxes()
+        self.timelineChart.removeAxis(self.timelineChart.axisX(timelineSeries))
+        self.timelineChart.removeAxis(self.timelineChart.axisY(timelineSeries))
+
+        yAxis = QtCharts.QValueAxis()
+        yAxis.applyNiceNumbers()
+        yAxis.setTickCount(min(maxEntityNum + 1, 4))
+
+        xAxis = QtCharts.QBarCategoryAxis()
+        xAxis.append(xAxisValues)
+        self.timelineChart.setAxisY(yAxis, timelineSeries)
+        self.timelineChart.setAxisX(xAxis, timelineSeries)
+
+        timesteps = len(timestep)
+        if timesteps == 0:
+            self.timescaleSelector.adjustLabelsToYear()
+        elif timesteps == 1:
+            self.timescaleSelector.adjustLabelsToMonth()
+        elif timesteps == 2:
+            self.timescaleSelector.adjustLabelsToDay()
+        elif timesteps == 3:
+            self.timescaleSelector.adjustLabelsToHour()
+        elif timesteps == 4:
+            self.timescaleSelector.adjustLabelsToMinute()
+        elif timesteps == 5:
+            self.timescaleSelector.adjustLabelsToSecond()
+        else:
+            # Just in case.
+            self.timescaleSelector.adjustLabelsToSecond()
+
+        self.mainWindow.timelineSelectMatchingEntities(timestep)
+
+
+class TimelineBarSet(QtCharts.QBarSet):
+
+    def __init__(self, label: str, timeWidget: TimeWidget, timestep: list, barSeriesDict: list):
+        super(TimelineBarSet, self).__init__(label)
+
+        self.clicked.connect(self.selectedAction)
+        self.timeWidget = timeWidget
+        self.timestep = timestep
+        self.barSeriesDict = barSeriesDict
+
+    def selectedAction(self, index):
+        self.timestep.append(self.barSeriesDict[index])
+        self.timeWidget.drawChart(self.timestep)
+
+
+class TimelineTimescaleSelector(QtWidgets.QLabel):
+
+    def __init__(self, timeWidget: TimeWidget):
+        super(TimelineTimescaleSelector, self).__init__(parent=timeWidget)
+        self.setStyleSheet("""border: 2px solid rgb(44, 49, 58);""")
+
+        self.timeWidget = timeWidget
+        self.setMinimumWidth(150)
+        self.setMaximumHeight(150)
+
+        self.setLayout(QtWidgets.QFormLayout())
+        self.setFrameStyle(QtWidgets.QFrame.Sunken)
+
+        self.yearButton = QtWidgets.QPushButton(' Year: ')
+        self.yearButton.clicked.connect(self.yearButtonPressed)
+        self.yearText = QtWidgets.QLabel('-')
+        self.yearText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.monthButton = QtWidgets.QPushButton(' Month: ')
+        self.monthButton.clicked.connect(self.monthButtonPressed)
+        self.monthText = QtWidgets.QLabel('X')
+        self.monthText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.dayButton = QtWidgets.QPushButton(' Day: ')
+        self.dayButton.clicked.connect(self.dayButtonPressed)
+        self.dayText = QtWidgets.QLabel('X')
+        self.dayText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.hourButton = QtWidgets.QPushButton(' Hour: ')
+        self.hourButton.clicked.connect(self.hourButtonPressed)
+        self.hourText = QtWidgets.QLabel('X')
+        self.hourText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.minuteButton = QtWidgets.QPushButton(' Minute: ')
+        self.minuteButton.clicked.connect(self.minuteButtonPressed)
+        self.minuteText = QtWidgets.QLabel('X')
+        self.minuteText.setFrameStyle(QtWidgets.QFrame.Sunken)
+
+        self.layout().addRow(self.yearButton, self.yearText)
+        self.layout().addRow(self.monthButton, self.monthText)
+        self.layout().addRow(self.dayButton, self.dayText)
+        self.layout().addRow(self.hourButton, self.hourText)
+        self.layout().addRow(self.minuteButton, self.minuteText)
+
+        self.yearButton.setDown(True)
+        self.yearButton.setDisabled(True)
+        self.monthButton.setDisabled(True)
+        self.dayButton.setDisabled(True)
+        self.hourButton.setDisabled(True)
+        self.minuteButton.setDisabled(True)
+
+    def yearButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:0])
+        self.adjustLabelsToYear()
+
+    def adjustLabelsToYear(self):
+        self.yearButton.setDown(True)
+        self.yearButton.setDisabled(True)
+        self.monthButton.setDisabled(True)
+        self.monthButton.setDown(False)
+        self.dayButton.setDisabled(True)
+        self.dayButton.setDown(False)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText('-')
+        self.monthText.setText('X')
+        self.dayText.setText('X')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def monthButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:1])
+        self.adjustLabelsToMonth()
+
+    def adjustLabelsToMonth(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(True)
+        self.monthButton.setDisabled(True)
+        self.dayButton.setDisabled(True)
+        self.dayButton.setDown(False)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText('-')
+        self.dayText.setText('X')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def dayButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:2])
+        self.adjustLabelsToDay()
+
+    def adjustLabelsToDay(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(True)
+        self.dayButton.setDisabled(True)
+        self.hourButton.setDisabled(True)
+        self.hourButton.setDown(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText('-')
+        self.hourText.setText('X')
+        self.minuteText.setText('X')
+
+    def hourButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:3])
+        self.adjustLabelsToHour()
+
+    def adjustLabelsToHour(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(True)
+        self.hourButton.setDisabled(True)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(True)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText('-')
+        self.minuteText.setText('X')
+
+    def minuteButtonPressed(self):
+        self.timeWidget.drawChart(self.timeWidget.currentTimeStep[:4])
+        self.adjustLabelsToMinute()
+
+    def adjustLabelsToMinute(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(False)
+        self.hourButton.setDisabled(False)
+        self.minuteButton.setDown(True)
+        self.minuteButton.setDisabled(True)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText(str(self.timeWidget.currentTimeStep[3]))
+        self.minuteText.setText('-')
+
+    def adjustLabelsToSecond(self):
+        self.yearButton.setDown(False)
+        self.yearButton.setDisabled(False)
+        self.monthButton.setDown(False)
+        self.monthButton.setDisabled(False)
+        self.dayButton.setDown(False)
+        self.dayButton.setDisabled(False)
+        self.hourButton.setDown(False)
+        self.hourButton.setDisabled(False)
+        self.minuteButton.setDown(False)
+        self.minuteButton.setDisabled(False)
+        self.yearText.setText(str(self.timeWidget.currentTimeStep[0]))
+        self.monthText.setText(str(self.timeWidget.currentTimeStep[1]))
+        self.dayText.setText(str(self.timeWidget.currentTimeStep[2]))
+        self.hourText.setText(str(self.timeWidget.currentTimeStep[3]))
+        self.minuteText.setText(str(self.timeWidget.currentTimeStep[4]))
+
+
+class ServerStatusBox(QtWidgets.QLabel):
+
+    def __init__(self, parent):
+        super(ServerStatusBox, self).__init__(parent=parent)
+        self.setAlignment(QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
+        self.setFrameStyle(QtWidgets.QFrame.Sunken | QtWidgets.QFrame.StyledPanel)
+        self.setText("Not connected to a server")
+
+    def updateStatus(self, status: str):
+        self.setText(status)
+
+
+class ChatBox(QtWidgets.QWidget):
+    """
+    Sends chat messages to everyone connected to the same server.
+    """
+
+    def receiveMessage(self, message: str):
+        self.textView.appendPlainText(message)
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.setMinimumWidth(500)
+        self.setMaximumWidth(500)
+
+        self.chatName = getuser() + ": "
+
+        chatLayout = QtWidgets.QGridLayout()
+        self.setLayout(chatLayout)
+
+        chatLabel = QtWidgets.QLabel('Project Collaboration Chat')
+        chatLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
+        chatLabel.setAlignment(QtCore.Qt.AlignCenter)
+        self.textView = QtWidgets.QPlainTextEdit()
+        self.textView.setReadOnly(True)
+        self.textSendBox = QtWidgets.QLineEdit()
+        self.textSendBox.setPlaceholderText("Type a message to send...")
+        self.textSendButton = QtWidgets.QPushButton(" Send ")
+        self.textSendButton.clicked.connect(self.sendMessage)
+
+        chatLayout.addWidget(chatLabel, 0, 0, 1, 2)
+        chatLayout.addWidget(self.textView, 1, 0, 1, 2)
+        chatLayout.addWidget(self.textSendBox, 2, 0)
+        chatLayout.addWidget(self.textSendButton, 2, 1)
+
+    def sendMessage(self):
+        self.mainWindow.sendChatMessage(self.chatName + self.textSendBox.text())
+        self.receiveMessage(self.chatName + self.textSendBox.text())
+        self.textSendBox.setText("")
+
+
+class LoggingUpdateThread(QtCore.QThread):
+    loggingSignal = QtCore.Signal(str)
+    endLogging = False
+
+    def __init__(self, messageHandler):
+        super().__init__()
+        self.messageHandler = messageHandler
+
+    def run(self):
+        while True:
+            if self.endLogging:
+                break
+            if not self.messageHandler.logQueue.empty():
+                try:
+                    logMsg = self.messageHandler.logQueue.get().getMessage()
+                    self.loggingSignal.emit(logMsg)
+                except queue.Empty:
+                    pass
+            else:
+                self.msleep(100)

Core/Interface/DockBarTwo.py

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+
+from PySide6 import QtWidgets, QtCore, QtGui
+from Core.Interface import Stylesheets
+
+
+class DockBarTwo(QtWidgets.QDockWidget):
+
+    def initialiseLayout(self) -> None:
+        childWidget = QtWidgets.QTabWidget()
+        self.setWidget(childWidget)
+        scrollAreaWidget = QtWidgets.QScrollArea()
+
+        scrollAreaWidget.setWidget(self.entDetails)
+        scrollAreaWidget.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
+
+        childWidget.addTab(scrollAreaWidget, 'Entity Details')
+        childWidget.addTab(self.oracle, 'Oracle')
+        self.setMaximumWidth(500)
+        self.resize(self.height(), 500)
+
+    def __init__(self,
+                 mainWindow,
+                 resourceHandler,
+                 entityDB,
+                 title="DockBar Two"):
+        super(DockBarTwo, self).__init__(parent=mainWindow)
+
+        self.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
+                             QtCore.Qt.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setWindowTitle(title)
+        self.resourceHandler = resourceHandler
+        self.entityDB = entityDB
+        self.setObjectName(title)
+
+        self.oracle = Oracle(self.parent(), self)
+        self.entDetails = EntityDetails(self.resourceHandler,
+                                        self.entityDB,
+                                        self.parent(),
+                                        self)
+
+        self.initialiseLayout()
+
+
+class EntityDetails(QtWidgets.QWidget):
+    """
+    This class is used to display a detailed view of the attributes of
+    the selected entity / entities.
+    """
+
+    def __init__(self,
+                 resourceHandler,
+                 entityDB,
+                 mainWindow,
+                 parent=None):
+
+        super(EntityDetails, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        self.resourceHandler = resourceHandler
+        self.entityDB = entityDB
+        self.detailsLayout = QtWidgets.QStackedLayout()
+        self.setLayout(self.detailsLayout)
+
+        layoutNothingSelected = QtWidgets.QGridLayout()
+        widgetNothing = QtWidgets.QWidget()
+        widgetNothing.setLayout(layoutNothingSelected)
+        self.detailsLayout.addWidget(widgetNothing)
+
+        layoutOneNodeSelected = QtWidgets.QGridLayout()
+        widgetOneNode = QtWidgets.QWidget()
+        widgetOneNode.setLayout(layoutOneNodeSelected)
+        self.detailsLayout.addWidget(widgetOneNode)
+
+        layoutMultipleItemsSelected = QtWidgets.QGridLayout()
+        widgetMultiItems = QtWidgets.QWidget()
+        widgetMultiItems.setLayout(layoutMultipleItemsSelected)
+        self.detailsLayout.addWidget(widgetMultiItems)
+
+        # Need to keep track of how many nodes are selected.
+        # ~ Nothing Selected/Hovered Layout
+        nothingLabel = QtWidgets.QLabel("Nothing is Selected.")
+        nothingLabel.setAlignment(QtCore.Qt.AlignHCenter |
+                                  QtCore.Qt.AlignVCenter)
+        layoutNothingSelected.addWidget(nothingLabel)
+        ###
+
+        # ~ One Link/Node Selected/Hovered Layout
+        # ~~ Part 1
+        summaryLayout = QtWidgets.QGridLayout()
+        summaryPanel = QtWidgets.QWidget()
+        summaryPanel.setLayout(summaryLayout)
+        self.summaryIcon = QtWidgets.QLabel("")  # QtGui.QIcon(None)
+        self.entityTypeLabel = QtWidgets.QLabel("")
+        self.entityUIDLabel = QtWidgets.QLabel("")
+        self.entityPrimaryLabel = QtWidgets.QLabel("")
+        summaryLayout.addWidget(self.summaryIcon, 0, 0)
+        summaryLayout.addWidget(self.entityTypeLabel, 0, 1, 1, 3)
+        summaryLayout.addWidget(self.entityPrimaryLabel, 1, 1)
+        summaryLayout.addWidget(self.entityUIDLabel, 2, 1)
+        layoutOneNodeSelected.addWidget(summaryPanel, 0, 0)
+
+        # ~~ Part 2
+        self.nodeLinkL = QtWidgets.QStackedLayout()
+        nodeLinkSwitcher = QtWidgets.QWidget()
+        nodeLinkSwitcher.setLayout(self.nodeLinkL)
+        layoutOneNodeSelected.addWidget(nodeLinkSwitcher, 1, 0)
+
+        relationshipsLayout = QtWidgets.QGridLayout()
+        relationshipsPanel = QtWidgets.QWidget()
+        relationshipsPanel.setLayout(relationshipsLayout)
+        self.relationshipsIncomingTable = RelationshipsTable(self, mainWindow)
+        self.relationshipsOutgoingTable = RelationshipsTable(self, mainWindow)
+        self.relationshipsIncomingTable.setHeaderLabel('Incoming Links')
+        self.relationshipsOutgoingTable.setHeaderLabel('Outgoing Links')
+        relationshipsLayout.addWidget(self.relationshipsIncomingTable, 0, 0)
+        relationshipsLayout.addWidget(self.relationshipsOutgoingTable, 1, 0)
+        self.nodeLinkL.addWidget(relationshipsPanel)
+
+        oneLinkRelLayout = QtWidgets.QHBoxLayout()
+        oneLinkRelPanel = QtWidgets.QWidget()
+        oneLinkRelPanel.setMaximumHeight(150)
+        oneLinkRelPanel.setLayout(oneLinkRelLayout)
+        self.linkParent = SingleLinkItem(self, mainWindow)  # QtWidgets.QLabel("")
+        self.linkParent.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
+        self.linkIcon = QtWidgets.QLabel("")
+        self.linkIcon.setMaximumHeight(90)
+        self.linkIcon.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
+        self.linkIcon.setAlignment(QtCore.Qt.AlignCenter)
+        self.linkChild = SingleLinkItem(self, mainWindow)  # QtWidgets.QLabel("")
+        self.linkChild.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
+        oneLinkRelLayout.addWidget(self.linkParent)
+        oneLinkRelLayout.addWidget(self.linkIcon)
+        oneLinkRelLayout.addWidget(self.linkChild)
+        self.nodeLinkL.addWidget(oneLinkRelPanel)
+
+        # ~~ Part 3
+        scroll = QtWidgets.QScrollArea()
+
+        self.detailsLayoutOneNode = QtWidgets.QFormLayout()
+        detailsLayoutOneNodePanel = QtWidgets.QWidget()
+        detailsLayoutOneNodePanel.setLayout(self.detailsLayoutOneNode)
+        scroll.setWidget(detailsLayoutOneNodePanel)
+        scroll.setWidgetResizable(True)
+        scroll.setMinimumHeight(200)
+        # scroll.setMaximumHeight(400)
+        layoutOneNodeSelected.addWidget(scroll, 2, 0)
+
+        # ~ Multiple Links/Nodes Selected Layout
+        self.nodesTable = RelationshipsTable(self, mainWindow)
+        self.nodesTable.setMaximumHeight(300)
+        self.nodesTable.setHeaderLabels(["Entity", "Incoming Links", "Outgoing Links"])
+        self.linksTable = LinksTable(self, mainWindow)
+        self.linksTable.setMaximumHeight(300)
+        self.linksTable.setHeaderLabels(["Parent", "Child"])
+        multiNodesTableLabelOne = QtWidgets.QLabel("Selected Nodes:")
+        layoutMultipleItemsSelected.addWidget(multiNodesTableLabelOne, 0, 0)
+        layoutMultipleItemsSelected.addWidget(self.nodesTable, 1, 0)
+        multiNodesTableLabelTwo = QtWidgets.QLabel("Selected Links:")
+        layoutMultipleItemsSelected.addWidget(multiNodesTableLabelTwo, 2, 0)
+        layoutMultipleItemsSelected.addWidget(self.linksTable, 3, 0)
+
+        self.setMinimumWidth(475)
+        self.setMaximumWidth(475)
+        self.setMaximumHeight(550)
+
+        self.currentlyShown = []
+
+    def displayWidgetDetails(self, jsonDicts) -> None:
+        """
+        jsonDicts: a list of json strings, each representing
+                   an entity or link.
+        isNode: Whether the json string(s) is/are nodes or links.
+        """
+        if jsonDicts == self.currentlyShown:
+            return
+        self.currentlyShown = jsonDicts
+        numberOfItems = len(jsonDicts)
+        if numberOfItems == 0:
+            self.layout().setCurrentIndex(0)
+            return
+        if None in jsonDicts:
+            self.mainWindow.MESSAGEHANDLER.error('Received None value when trying to display widget details.',
+                                                 popUp=False)
+            return
+        self.clearDetailsHelper()
+        isNode = None
+        if numberOfItems == 1:
+            entity = jsonDicts[0]
+            isNode = self.entityDB.isNode(entity['uid'])
+            self.populateOneSummaryHelper(entity, isNode)
+            self.populateOneDetailsHelper(entity)
+            if isNode:
+                self.populateOneRelationshipHelper(entity['uid'])
+            else:
+                self.populateOneLinkRelationshipHelper(entity['uid'])
+        else:
+            for item in jsonDicts:
+                if item.get('Entity Type') is not None:
+                    self.populateMultiRelationshipHelperNode(item)
+                else:
+                    self.populateMultiRelationshipHelperLink(item)
+        self.switchLayoutHelper(numberOfItems, isNode)
+
+    # Display helper functions
+    def clearDetailsHelper(self) -> None:
+        """
+        Clear tree widget items from tree widgets.
+        """
+        rows = self.detailsLayoutOneNode.rowCount()
+        for row in range(rows):
+            self.detailsLayoutOneNode.removeRow(0)
+        self.relationshipsIncomingTable.clear()
+        self.relationshipsOutgoingTable.clear()
+        self.nodesTable.clear()
+        self.linksTable.clear()
+
+    def populateOneDetailsHelper(self, jsonDict) -> None:
+        if jsonDict is None or jsonDict == []:
+            return
+        for key in jsonDict:
+            if key == "uid" or key == "Child UIDs" or key == "Icon":
+                continue
+            elif key == "Notes":
+                notesTextArea = QtWidgets.QPlainTextEdit(jsonDict[key])
+                notesTextArea.setReadOnly(True)
+                self.detailsLayoutOneNode.addRow(key, notesTextArea)
+            else:
+                self.detailsLayoutOneNode.addRow(key, QtWidgets.QLabel(str(jsonDict[key])))
+
+    def populateOneLinkRelationshipHelper(self, uid) -> None:
+        first = uid[0]
+        firstJson = self.entityDB.getEntity(first)
+        second = uid[1]
+        secondJson = self.entityDB.getEntity(second)
+        firstPixmap = QtGui.QPixmap()
+        firstPixmap.loadFromData(firstJson.get('Icon'))
+        secondPixmap = QtGui.QPixmap()
+        secondPixmap.loadFromData(secondJson.get('Icon'))
+        self.linkParent.linkItemPic.setPixmap(firstPixmap)
+        self.linkParent.linkItemName.setText(firstJson[list(firstJson)[1]])
+        self.linkParent.linkItemUid = firstJson['uid']
+        self.linkChild.linkItemPic.setPixmap(secondPixmap)
+        self.linkChild.linkItemName.setText(secondJson[list(secondJson)[1]])
+        self.linkChild.linkItemUid = secondJson['uid']
+
+    def populateOneRelationshipHelper(self, uid) -> None:
+        inc = self.entityDB.getIncomingLinks(uid)
+        out = self.entityDB.getOutgoingLinks(uid)
+        for edge in inc:
+            uid = edge[0]
+            edgeJson = self.entityDB.getEntity(uid)
+            nodePixmap = QtGui.QPixmap()
+            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            ResolutionTreeWidgetEntity(self.relationshipsIncomingTable,
+                                       nodePixmap,
+                                       edgeJson[list(edgeJson)[1]],
+                                       uid)
+        for edge in out:
+            uid = edge[1]
+            edgeJson = self.entityDB.getEntity(uid)
+            nodePixmap = QtGui.QPixmap()
+            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            ResolutionTreeWidgetEntity(self.relationshipsOutgoingTable,
+                                       nodePixmap,
+                                       edgeJson[list(edgeJson)[1]],
+                                       uid)
+
+    def populateMultiRelationshipHelperNode(self, nodeJson) -> None:
+        inc = len(self.entityDB.getIncomingLinks(nodeJson['uid']))
+        out = len(self.entityDB.getOutgoingLinks(nodeJson['uid']))
+        nodePixmap = QtGui.QPixmap()
+        nodePixmap.loadFromData(nodeJson.get('Icon'))
+        ResolutionTreeWidgetEntity(self.nodesTable,
+                                   nodePixmap,
+                                   nodeJson[list(nodeJson)[1]],
+                                   nodeJson['uid'],
+                                   inc,
+                                   out)
+
+    def populateMultiRelationshipHelperLink(self, linkJson) -> None:
+        uid = linkJson['uid']
+        parent = self.entityDB.getEntity(uid[0])
+        child = self.entityDB.getEntity(uid[1])
+        LinksTreeWidgetEntity(self.linksTable,
+                              QtGui.QPixmap(self.resourceHandler.getLinkPicture()),
+                              parent[list(parent)[1]],
+                              child[list(child)[1]],
+                              uid)
+
+    def populateOneSummaryHelper(self, jsonDict, isNode) -> None:
+        if isNode:
+            self.entityUIDLabel.setText(str(jsonDict[list(jsonDict)[1]]))
+            self.entityPrimaryLabel.setText(jsonDict[list(jsonDict)[0]])
+            self.entityTypeLabel.setText(jsonDict['Entity Type'])
+            summaryPixmap = QtGui.QPixmap()
+            summaryPixmap.loadFromData(jsonDict.get('Icon'))
+            self.summaryIcon.setPixmap(summaryPixmap)
+        else:
+            self.entityUIDLabel.setText("--")
+            self.entityPrimaryLabel.setText(jsonDict['Resolution'])
+            self.entityTypeLabel.setText('Resolution')
+            self.summaryIcon.setPixmap(QtGui.QPixmap(
+                self.resourceHandler.getLinkPicture()))
+            self.linkIcon.setPixmap(QtGui.QPixmap(
+                self.resourceHandler.getLinkArrowPicture()))
+
+    def switchLayoutHelper(self, selectionCount, isNode) -> None:
+        """
+        Switches between showing nothing, showing details for 1 node/link
+           and showing details for multiple nodes/links
+        Layout 0 = Nothing
+        Layout 1 = 1 node / link
+        Layout 2 = multi nodes / links
+        """
+        # selectionCount should never be 0, it is checked before this function is called
+        if selectionCount == 1:
+            if isNode:
+                self.detailsLayout.setCurrentIndex(1)
+                self.nodeLinkL.setCurrentIndex(0)
+            else:
+                self.detailsLayout.setCurrentIndex(1)
+                self.nodeLinkL.setCurrentIndex(1)
+        else:
+            self.detailsLayout.setCurrentIndex(2)
+
+
+class SingleLinkItem(QtWidgets.QWidget):
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+
+        self.linkItemPic = QtWidgets.QLabel()
+        
+        self.linkItemPic.setAlignment(QtCore.Qt.AlignCenter)
+        self.linkItemName = QtWidgets.QLabel()
+        
+        self.linkItemName.setAlignment(QtCore.Qt.AlignCenter)
+        self.linkItemUid = ""
+        self.setMaximumHeight(90)
+
+        linkItemLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(linkItemLayout)
+        linkItemLayout.addWidget(self.linkItemPic)
+        linkItemLayout.addWidget(self.linkItemName)
+        self.mainWindow = mainWindow
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        self.mainWindow.setCurrentCanvasSelection([self.linkItemUid])
+        super().mousePressEvent(event)
+
+
+class RelationshipsTable(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+        self.mainWindow = mainWindow
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        if event.button() == QtGui.Qt.MouseButton.LeftButton:
+            itemClicked = self.itemAt(event.pos())
+            if itemClicked is None:
+                return
+            self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
+        else:
+            super().mousePressEvent(event)
+
+
+class LinksTable(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindow):
+        super().__init__(parent=parent)
+        self.mainWindow = mainWindow
+
+    def mousePressEvent(self, event):
+        """
+        Have the canvas select the clicked item.
+        """
+        if event.button() == QtGui.Qt.MouseButton.LeftButton:
+            itemClicked = self.itemAt(event.pos())
+            if itemClicked is None:
+                return
+            self.mainWindow.setCurrentCanvasSelection([itemClicked.uid])
+        else:
+            super().mousePressEvent(event)
+
+
+class LinksTreeWidgetEntity(QtWidgets.QTreeWidgetItem):
+    def __init__(self,
+                 parent,
+                 icon=None,
+                 ent1=None,
+                 ent2=None,
+                 uid=None):
+        super().__init__(parent)
+
+        self.setText(0, str(ent1))
+        self.setText(1, str(ent2))
+        self.uid = uid
+        self.setIcon(0, icon)
+        self.setIcon(1, icon)
+
+
+class ResolutionTreeWidgetEntity(QtWidgets.QTreeWidgetItem):
+    def __init__(self,
+                 parent,
+                 icon=None,
+                 name="",
+                 entityUID=None,
+                 relInc=None,
+                 relOut=None):
+
+        super().__init__(parent, [name])
+        if icon is not None:
+            self.setIcon(0, icon)
+        if relInc is not None:
+            self.setText(1, str(relInc))
+        if relOut is not None:
+            self.setText(2, str(relOut))
+        self.entityUID = entityUID
+
+
+class Oracle(QtWidgets.QWidget):
+    """
+    This class is used to query the Question Answering AI model.
+    It will be fed data from the files the user imports into
+      the software.
+    """
+
+    def __init__(self, mainWindow, parent=None):
+        super(Oracle, self).__init__(parent=parent)
+
+        self.mainWindow = mainWindow
+        oracleLayout = QtWidgets.QGridLayout()
+        self.setLayout(oracleLayout)
+
+        self.answerLabel = QtWidgets.QLabel("Answer Section")
+        self.answerLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
+        self.answerLabel.setAlignment(QtCore.Qt.AlignHCenter |
+                                      QtCore.Qt.AlignVCenter)
+        oracleLayout.addWidget(self.answerLabel, 1, 0, 1, 2)
+        self.answerSection = QtWidgets.QPlainTextEdit()
+        self.answerSection.setReadOnly(True)
+        self.answerSection.setPlaceholderText(
+            "The answer to your Question will appear here.")
+        self.answerSection.setUndoRedoEnabled(False)
+        oracleLayout.addWidget(self.answerSection, 2, 0, 1, 2)
+
+        self.questionLabel = QtWidgets.QLabel("Ask a Question")
+        self.questionLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
+        self.questionLabel.setAlignment(QtCore.Qt.AlignHCenter |
+                                        QtCore.Qt.AlignVCenter)
+        oracleLayout.addWidget(self.questionLabel, 3, 0, 1, 2)
+        self.questionSection = QtWidgets.QLineEdit()
+        self.questionSection.setPlaceholderText("Ask a Question here.")
+
+        oracleLayout.addWidget(self.questionSection, 4, 0)
+
+        self.submitQuestionButton = QtWidgets.QPushButton("Ask")
+        oracleLayout.addWidget(self.submitQuestionButton, 4, 1)
+        self.questionSection.setMinimumWidth(200)
+
+        self.submitQuestionButton.clicked.connect(mainWindow.askQuestion)

Core/Interface/Entity.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python3
+
+from json import dumps
+import math
+from typing import Any, Optional
+
+from PySide6 import QtCore, QtGui, QtWidgets
+from PySide6.QtWidgets import QGraphicsItem
+from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem
+from PySide6.QtSvgWidgets import QGraphicsSvgItem
+
+
+class BaseNode(QGraphicsItemGroup):
+
+    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str) -> None:
+        super(BaseNode, self).__init__()
+
+        self.setCacheMode(self.DeviceCoordinateCache)
+
+        self.pixmapItem = QtGui.QPixmap()
+        self.pixmapItem.loadFromData(pictureByteArray)
+
+        if pictureByteArray.data().startswith(b'<svg '):
+            self.iconItem = QGraphicsSvgItem()
+            self.iconItem.renderer().load(pictureByteArray)
+            # Force recalculation of geometry, else this looks like 1 pixel.
+            # https://stackoverflow.com/a/68182093
+            self.iconItem.setElementId("")
+        else:
+            self.iconItem = QGraphicsPixmapItem(self.pixmapItem)
+
+        self.iconItem.setCacheMode(self.iconItem.DeviceCoordinateCache, QtCore.QSize(40, 40))
+
+        self.labelItem = QGraphicsSimpleTextItem('')
+        self.addToGroup(self.iconItem)
+        self.addToGroup(self.labelItem)
+
+        self.updateLabel(primaryAttribute)
+        self.labelItem.setCacheMode(self.labelItem.DeviceCoordinateCache)
+
+        self.uid = uid
+        self.setFlag(QGraphicsItem.ItemIsMovable, True)
+        self.setFlag(QGraphicsItem.ItemIsSelectable, True)
+        # Check if this option makes the software feel better or worse to use.
+        self.setFlag(QGraphicsItem.ItemClipsToShape, True)
+        self.setAcceptHoverEvents(True)
+
+        self.connectors = []
+        self.bookmarked = False
+        self.isBeingResolved = False
+        self.parentGroup = None
+
+    def updateLabel(self, newText: str = '') -> None:
+        if newText != '':
+            self.labelItem.setText(newText)
+            labelWidth = self.labelItem.boundingRect().width()
+            self.labelItem.setPos(self.iconItem.x() + 20 - (labelWidth / 2), self.iconItem.y() + 45)
+
+    def removeConnector(self, connector) -> None:
+        # Exception could be thrown if the connector is already deleted.
+        try:
+            self.connectors.remove(connector)
+        except ValueError:
+            pass
+
+    def addConnector(self, connector) -> None:
+        self.connectors.append(connector)
+
+    def getConnectorByUID(self, uid) -> None:
+        for connector in self.connectors:
+            if connector.uid == uid:
+                return connector
+
+    def hoverEnterEvent(self, event) -> None:
+        super().hoverEnterEvent(event)
+        self.scene().detailsWidgetCaller(uid=self.uid)
+
+    def hoverLeaveEvent(self, event) -> None:
+        super().hoverLeaveEvent(event)
+        self.scene().detailsWidgetCaller()
+
+    def mouseDoubleClickEvent(self, event) -> None:
+        self.scene().editEntityProperties(self.uid)
+        super().mouseDoubleClickEvent(event)
+
+    def shape(self) -> QtGui.QPainterPath:
+        return self.iconItem.shape()
+
+    def boundingRect(self) -> QtCore.QRectF:
+        return self.iconItem.boundingRect()
+
+    def childrenBoundingRect(self) -> QtCore.QRectF:
+        return self.iconItem.boundingRect()
+
+    def boundingRegion(self, itemToDeviceTransform: QtGui.QTransform) -> QtGui.QRegion:
+        return self.iconItem.boundingRegion(itemToDeviceTransform)
+
+
+class GroupNode(BaseNode):
+
+    # childNodes is a list of tuples, uid and picture, of all the nodes in the group.
+    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group') -> None:
+        super(GroupNode, self).__init__(pictureByteArray, uid, label)
+        self.groupedNodesConnectors = []
+        self.itemsThatWereGrouped = []
+        self.groupedNodesUid = set()
+
+        self.listWidget = GroupNodeChildList()
+        self.listProxyWidget = None
+
+    def itemChange(self, change: QtWidgets.QGraphicsItem.GraphicsItemChange, value: Any) -> Any:
+        if change == QtWidgets.QGraphicsItem.ItemSelectedChange:
+            if value:
+                self.showList(None)
+            else:
+                self.hideList()
+        return value
+
+    def showList(self, pos: QtCore.QPoint = None) -> None:
+        if self.listWidget is None or self.listProxyWidget is None:
+            return
+        self.listWidget.itemList.clear()
+        for uid in self.groupedNodesUid:
+            entityJson = self.scene().parent().entityDB.getEntity(uid)
+            try:
+                primaryField = entityJson[list(entityJson)[1]]
+            except IndexError:
+                primaryField = ''
+            iconPixmap = QtGui.QPixmap()
+            iconPixmap.loadFromData(entityJson['Icon'])
+
+            GroupNodeListItem(icon=iconPixmap, text=primaryField, uid=uid,
+                              listview=self.listWidget.itemList)
+        if pos is None:
+            pos = QtCore.QPoint(self.pos().x() + 60, self.pos().y())
+        self.listProxyWidget.setPos(pos)
+        self.listProxyWidget.setVisible(True)
+
+    def mouseMoveEvent(self, event: QtWidgets.QGraphicsSceneMouseEvent) -> None:
+        super(GroupNode, self).mouseMoveEvent(event)
+        if self.listProxyWidget is not None and self.listProxyWidget.isVisible():
+            self.listProxyWidget.setPos(QtCore.QPoint(self.pos().x() + 60, self.pos().y()))
+
+    def hideList(self) -> None:
+        if self.listProxyWidget is not None:
+            self.listProxyWidget.hide()
+            self.listProxyWidget.setVisible(False)
+
+    def formGroup(self, childNodeUIDs, listProxyWidget: QtWidgets.QGraphicsProxyWidget) -> None:
+        [self.addItemToGroup(uid) for uid in childNodeUIDs]  # Should be faster than just a for loop
+        self.listProxyWidget = listProxyWidget
+        self.listProxyWidget.setCacheMode(self.DeviceCoordinateCache)
+
+    def addItemToGroup(self, uid: str) -> None:
+        self.groupedNodesUid.add(uid)
+        self.handleAddItemConnectors(uid)
+
+    def handleAddItemConnectors(self, uid) -> None:
+        incomingLinks = self.scene().parent().entityDB.getIncomingLinks(uid)
+        outgoingLinks = self.scene().parent().entityDB.getOutgoingLinks(uid)
+
+        [self.scene().addLinkProgrammatic(link[0], self.uid) for link in incomingLinks]
+        [self.scene().addLinkProgrammatic(self.uid, link[1]) for link in outgoingLinks]
+
+    def removeSpecificItemFromGroupIfExists(self, uid) -> bool:
+        if uid in self.groupedNodesUid:
+            self.groupedNodesUid.remove(uid)
+            self.setSelected(False)
+            return True
+        return False
+
+
+# Ref: https://doc.qt.io/qt-5/qtwidgets-graphicsview-diagramscene-example.html#arrow-class-definition
+# Ref: https://github.com/PySide/Examples/blob/master/examples/graphicsview/diagramscene/diagramscene.py
+class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
+
+    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None) -> None:
+        super(BaseConnector, self).__init__(parent)
+        self.setCacheMode(self.ItemCoordinateCache)
+
+        self.myStartItem = origin
+        self.myEndItem = destination
+
+        self.labelItem = QGraphicsSimpleTextItem('')
+        self.labelItem.setCacheMode(self.labelItem.ItemCoordinateCache)
+        self.addToGroup(self.labelItem)
+
+        self.updateLabel(name)
+
+        if uid is not None:
+            if isinstance(uid, list) or isinstance(uid, set):
+                self.uid = set(uid)
+            else:
+                self.uid = {uid}
+        else:
+            self.uid = {(origin.uid, destination.uid)}
+
+        self.setFlag(QGraphicsItem.ItemIsMovable, False)
+        self.setFlag(QGraphicsItem.ItemIsSelectable, True)
+        self.labelItem.setFlag(QGraphicsItem.ItemIsSelectable, False)
+
+        self.setZValue(-100)
+        self.myStartItem.addConnector(self)
+        self.myEndItem.addConnector(self)
+
+        # Set as the wrong positions to force drawing.
+        self.oldStartPos = QtCore.QPointF(self.myStartItem.pos().x() + 1, 0)
+        self.oldEndPos = QtCore.QPointF(self.myEndItem.pos().x() + 1, 0)
+
+        self.colorSelected = QtGui.QColor(0, 173, 238)
+        self.colorDefault = QtGui.QColor(200, 200, 200)
+        self.myColor = self.colorDefault
+
+        self.pen = QtGui.QPen(self.myColor, 2, QtCore.Qt.SolidLine,
+                              QtCore.Qt.RoundCap, QtCore.Qt.RoundJoin)
+
+        self.arrowHead = QtGui.QPolygonF()
+        self.line = QtCore.QLineF()
+
+    def updateLabel(self, newText: str = '') -> None:
+        self.labelItem.setText(newText)
+        self.update()
+
+    def mousePressEvent(self, event) -> None:
+        self.scene().detailsWidgetCaller(uid=self.uid)
+        super().mousePressEvent(event)
+
+    def mouseDoubleClickEvent(self, event) -> None:
+        if len(self.uid) == 1:
+            self.scene().editLinkProperties(next(iter(self.uid)))
+        else:
+            # This will cause a message to be shown to the user, telling them that links representing multiple
+            #   connections cannot be edited.
+            self.scene().editLinkProperties("")
+        super().mouseDoubleClickEvent(event)
+
+    def startItem(self) -> BaseNode:
+        return self.myStartItem
+
+    def endItem(self) -> BaseNode:
+        return self.myEndItem
+
+    def updatePosition(self) -> None:
+        self.line = QtCore.QLineF(self.mapFromItem(self.myStartItem, 0, 0), self.mapFromItem(self.myEndItem, 0, 0))
+        self.update()
+
+    def boundingRect(self) -> QtCore.QRectF:
+        extra = (self.pen.width() + 20) / 2.0
+        p1 = self.line.p1()
+        p2 = self.line.p2()
+        return QtCore.QRectF(p1, QtCore.QSizeF(p2.x() - p1.x(), p2.y() - p1.y())
+                             ).normalized().adjusted(-extra, -extra, extra, extra)
+
+    def shape(self) -> QtGui.QPainterPath:
+        path = QtGui.QPainterPath(self.line.p1())
+        path.lineTo(self.line.p2())
+        path.addPolygon(self.arrowHead)
+        return path
+
+    def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
+              widget: Optional[QtWidgets.QWidget] = ...) -> None:
+
+        currentStartPos = self.myStartItem.pos()
+        currentEndPos = self.myEndItem.pos()
+
+        self.myColor = self.colorSelected if self.isSelected() else self.colorDefault
+
+        if currentEndPos == self.oldEndPos and currentStartPos == self.oldStartPos:
+            myPen = QtGui.QPen(self.myColor)
+            painter.setPen(myPen)
+            painter.setBrush(self.myColor)
+            painter.drawLine(self.line)
+            painter.drawPolygon(self.arrowHead)
+            return
+
+        self.oldStartPos = currentStartPos
+        self.oldEndPos = currentEndPos
+
+        p1 = QtCore.QPointF(self.myStartItem.pos().x() + 20, self.myStartItem.pos().y() + 20)
+        p2 = QtCore.QPointF(self.myEndItem.pos().x() + 20, self.myEndItem.pos().y() + 20)
+
+        line = QtCore.QLineF(p1, p2)
+
+        if line.length() < 45:
+            self.labelItem.hide()
+            return
+
+        angle = math.atan2(line.dy(), - line.dx())
+
+        if line.length() < 50 + len(self.labelItem.text()) * 15:
+            self.labelItem.hide()
+        else:
+            self.labelItem.show()
+            angle2 = math.degrees(math.pi - angle)
+            if 90 < angle2 < 270:
+                angle2 -= 180
+                # When the label flips, it starts from 0.3 and reads towards the starting node (as opposed to
+                #   reading towards the ending node).
+                # To avoid overlap, we get the proportion of the text to the length of the line, and add it to the
+                #   offset of 0.3 to keep the label at the same relative location.
+                self.labelItem.setPos(line.pointAt(0.3 + (self.labelItem.boundingRect().width() / line.length())))
+            else:
+                self.labelItem.setPos(line.pointAt(0.3))
+            self.labelItem.setRotation(angle2)
+
+        myPen = QtGui.QPen(self.myColor)
+        arrowSize = 20.0
+        painter.setPen(myPen)
+        painter.setBrush(self.myColor)
+
+        line.setLength(line.length() - 45)
+        self.line = line
+
+        arrowP1 = line.p2() + QtCore.QPointF(
+            math.sin(angle + math.pi / 3.0) * arrowSize,
+            math.cos(angle + math.pi / 3.0) * arrowSize)
+        arrowP2 = line.p2() + QtCore.QPointF(
+            math.sin(angle + math.pi - math.pi / 3.0) * arrowSize,
+            math.cos(angle + math.pi - math.pi / 3.0) * arrowSize)
+
+        self.arrowHead.clear()
+        for point in [line.p2(), arrowP1, arrowP2]:
+            self.arrowHead.append(point)
+
+        painter.drawLine(line)
+        painter.drawPolygon(self.arrowHead)
+
+
+class GroupNodeListItem(QtWidgets.QListWidgetItem):
+
+    def __init__(self, icon, text, uid: str, listview=None) -> None:
+        super(GroupNodeListItem, self).__init__(icon, text, listview)
+        self.uid = uid
+
+
+class GroupNodeChildList(QtWidgets.QWidget):
+
+    def __init__(self) -> None:
+        super(GroupNodeChildList, self).__init__()
+
+        self.setLayout(QtWidgets.QVBoxLayout())
+        titleLabel = QtWidgets.QLabel('Child Items')
+        titleLabel.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignHCenter)
+        self.layout().addWidget(titleLabel)
+
+        self.itemList = ChildListWidget()
+        self.layout().addWidget(self.itemList)
+
+
+class ChildListWidget(QtWidgets.QListWidget):
+
+    def __init__(self) -> None:
+        super(ChildListWidget, self).__init__()
+        self.setAlternatingRowColors(True)
+
+    def mouseMoveEvent(self, event: QtGui.QMouseEvent) -> None:
+        itemDragged = self.itemAt(event.pos())
+
+        if itemDragged is None:
+            return
+
+        self.setCurrentItem(itemDragged)
+        drag = QtGui.QDrag(self)
+        mimeData = QtCore.QMimeData()
+
+        mimeData.setText(dumps({'uid': itemDragged.uid}))
+        drag.setMimeData(mimeData)
+
+        # All Entities should have icons, but you never know.
+        pixmap = None
+        if itemDragged.icon() is not None:
+            pixmap = itemDragged.icon().pixmap(100, 100)
+            drag.setPixmap(pixmap)
+        drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
+        drag.exec_()
+
+        super().mousePressEvent(event)

Core/Interface/Stylesheets.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+
+TOOLBAR_STYLESHEET = """QToolBar {background-color: rgb(41, 45, 62);
+                     color: rgba(248, 248, 242, 1) !important;
+                     font-family: Segoe UI;
+                     font-size: 13px;
+                     text-align: left;}
+                     
+                     QToolBar::separator {
+                     background-color: rgb(0, 173, 238);
+                     width: 3px;
+                     height: 3px;
+                     }
+                     """
+
+MAIN_WINDOW_STYLESHEET = """
+                        QWidget, QDialog{ 
+                        background-color: rgb(41, 45, 62);
+                         color: rgba(248, 248, 242, 1) !important;
+                         font-size: 15px;
+                         }
+                        QScrollBar:vertical {         
+                        background:rgb(44, 49, 58);
+                        width:7px;    
+                        margin: 0px 0px 0px 0px;
+                        }
+                        QScrollBar::handle:vertical {
+                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                            min-height: 0px;
+                        }
+                        QScrollBar:horizontal {         
+                            background:rgb(44, 49, 58);
+                            height:7px;    
+                            margin: 0px 0px 0px 0px;
+                        }
+                        QScrollBar::handle:horizontal {
+                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                        }
+
+                        QMenuBar { 
+                          color: #ffffff;
+                          background-color: rgb(33, 37, 43);
+                          font-size: 15px;
+                        }
+
+                        QPushButton {
+                        border: 2px solid rgb(0, 173, 238);
+                        border-radius: 6px;
+                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
+                        min-width: 50px;
+                        }
+
+                        QLabel { 
+                        border: 2px solid rgb(41, 45, 62);
+                        padding-left: 7px;
+                        border-left-color: rgb(0, 173, 238);
+                        }
+                        
+                        QLineEdit, QPlainTextEdit { 
+                        border: 0.5px solid rgb(0, 173, 238);
+                        border-radius: 6px;
+                        }
+                        
+                        QTabBar::tab {background: rgb(68, 66, 103);
+                        border: 2px solid rgb(41, 45, 62);
+                        border-radius: 3px;
+                        min-height: 3ex;
+                        }
+                        
+                        QTabBar::tab:selected {background: rgb(41, 45, 62);
+                        border: 2px solid rgb(41, 45, 62);
+                        min-height: 2.5ex;
+                        border-radius: 3px;
+                        border-top-color: rgb(0, 173, 238);
+                        }
+                        
+                        
+                        """
+
+DOCK_BAR_TWO_LINK = """
+                  QLabel { 
+                        border: 1px solid rgb(41, 45, 62);
+                        }
+                  """
+
+DOCK_BAR_LABEL = """
+                  QLabel { 
+                        border: 1px solid rgb(41, 45, 62);
+                        border-radius: 2px;
+                        border-bottom-color: rgb(0, 173, 238);
+                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                          stop: 1 rgb(0, 173, 238), stop: 0.85 rgb(41, 45, 62));
+                        }
+                  """
+
+DROPDOWN_MENU_STYLE = "background-color: rgb(41, 45, 62);" \
+                      "color: rgba(248, 248, 242, 1) !important;" \
+                      "border: 2px solid rgb(44, 49, 58);" \
+                      "border-left-color: rgb(189, 147, 249);" \
+                      "font-family: Segoe UI;" \
+                      "font-size: 13px;"
+
+TEXT_BOX_STYLESHEET = "QLineEdit, QLabel{" \
+                      "border-bottom: 1px solid rgb(0, 173, 238);" \
+                      "}"
+
+CHECK_BOX_STYLESHEET = "QCheckBox::indicator:unchecked" \
+                       "{" \
+                       "border: 0.5px solid rgb(0, 173, 238);" \
+                       "background: none;" \
+                       "}"
+
+RADIO_BUTTON_STYLESHEET = "QRadioButton::indicator:unchecked" \
+                          "{" \
+                          "border: 0.5px solid rgb(0, 173, 238);" \
+                          "background: none;" \
+                          "border-radius:          7px;" \
+                          "}"
+
+SETTINGS_WIDGET_STYLESHEET = "#settingsWidget {background-color:rgb(41, 45, 62);}"
+
+MENUS_STYLESHEET = "background-color: rgb(41, 45, 62);" \
+                   "color: rgba(248, 248, 242, 1) !important;" \
+                   "border: 1px solid rgb(44, 49, 58);" \
+                   "border-bottom: 1px solid rgb(0, 173, 238);" \
+                   "font-family: Segoe UI;" \
+                   "font-size: 13px;" \
+                   "text-align: left;"
+
+MENUS_STYLESHEET_2 = """QMenu::item{
+                         background-color: rgb(41, 45, 62);
+                         color: rgba(248, 248, 242, 1) !important;
+                         border-left: 1px solid rgb(0, 173, 238);
+                         padding-right: 4px;
+                         padding-bottom: 4px;
+                         padding-top: 4px;
+                         font-size: 15px;
+                         text-align: left;
+                         padding-left: 4px;
+                         }
+                        
+                         QMenu::item:selected{
+                         background-color: rgb(0, 85, 127);
+                         color: rgb(255, 255, 255);
+                         }
+                         
+                         QMenu::item:disabled {
+                         background-color:rgb(81, 87, 114);
+                         }"""
+
+MERGE_STYLESHEET = """
+                QWidget, QDialog{ 
+                    background-color: rgb(41, 45, 62);
+                    color: rgba(248, 248, 242, 1) !important;
+                    font-family: Segoe UI;
+                    font-size: 13px;}
+                   
+                   QScrollBar:vertical {         
+                        background:rgb(44, 49, 58);
+                        width:7px;    
+                        margin: 0px 0px 0px 0px;
+                        }
+                        QScrollBar::handle:vertical {
+                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                            min-height: 0px;
+                        }
+                        QScrollBar:horizontal {         
+                            background:rgb(44, 49, 58);
+                            height:7px;    
+                            margin: 0px 0px 0px 0px;
+                        }
+                        QScrollBar::handle:horizontal {
+                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
+                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
+                        }"""
+
+RESOLUTION_WIZARD_STYLESHEET = "background-color: rgb(41, 45, 62);" \
+                               "color: rgba(248, 248, 242, 1) !important;" \
+                               "padding-bottom: 5px;" \
+                               "font-family: Segoe UI;" \
+                               "font-size: 13px;"
+
+PATH_INPUT_STYLESHEET = "border: 2px solid rgb(44, 49, 58);" \
+                        "border-radius: 25px;" \
+                        "padding: 4px; " \
+                        "background-color: rgb(129, 133, 137);" \
+                        "color: rgba(248, 248, 242, 1) !important;"
+
+BUTTON_STYLESHEET = """
+                        QPushButton {
+                        border: 2px solid rgb(0, 173, 238);
+                        border-radius: 6px;
+                        color: rgba(248, 248, 242, 1) !important;
+                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
+                        min-width: 80px;
+                        }
+                        QPushButton:pressed {
+                            background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                              stop: 0 #dadbde, stop: 1 #f6f7fa);
+                        }
+                    """
+
+BUTTON_STYLESHEET_2 = """
+                        QPushButton {
+                        border: 2px solid rgb(0, 173, 238);
+                        border-radius: 6px;
+                        color: rgba(248, 248, 242, 1) !important;
+                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
+                        
+                        min-width: 300px;
+                        
+                        }
+                        QPushButton:pressed {
+                            background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
+                                                              stop: 0 #dadbde, stop: 1 #f6f7fa);
+                        }
+                    """
+
+SELECT_PROJECT_STYLESHEET = "background-color: rgb(41, 45, 62);" \
+                            "color: rgba(248, 248, 242, 1) !important;" \
+                            "border-color: rgb(0, 173, 238);"

Core/Interface/ToolBarOne.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+
+from PySide6 import QtWidgets, QtGui
+from Core.Interface import Stylesheets
+
+
+class ToolBarOne(QtWidgets.QToolBar):
+
+    def __init__(self, title, parent):
+        # Parent is (expected to be) mainWindow.
+        super().__init__(title, parent=parent)
+        self.setObjectName(title)
+        self.setToolButtonStyle(QtGui.Qt.ToolButtonTextUnderIcon)
+        self.setStyleSheet(Stylesheets.TOOLBAR_STYLESHEET)
+
+        newCanvas = QtGui.QAction("Add Canvas",
+                                  self,
+                                  statusTip="Create new Canvas or Open existing Canvas",
+                                  triggered=self.addCanvas,
+                                  icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('addCanvas')))
+        self.addAction(newCanvas)
+        self.insertSeparator(newCanvas)
+
+        manualLink = QtGui.QAction('Create Manual &Link',
+                                   self,
+                                   statusTip="Link two entities together manually",
+                                   triggered=self.createManualLink,
+                                   icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('drawLink')))
+        self.addAction(manualLink)
+        self.insertSeparator(manualLink)
+
+        uploadSelected = QtGui.QAction('Upload Selected Files',
+                                       self,
+                                       statusTip="Upload the selected file entities to the server",
+                                       triggered=self.uploadSelectedFiles,
+                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('uploading')))
+        self.addAction(uploadSelected)
+        self.insertSeparator(uploadSelected)
+
+        selectLeaves = QtGui.QAction('Select Leaf Nodes',
+                                     self,
+                                     statusTip="Select nodes with one or more incoming links and no outgoing links.",
+                                     triggered=self.selectLeafNodes,
+                                     icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('leafNodes')))
+        self.addAction(selectLeaves)
+        self.insertSeparator(selectLeaves)
+
+        selectRoots = QtGui.QAction('Select Root Nodes',
+                                    self,
+                                    statusTip="Select nodes with one or more outgoing links and no incoming links.",
+                                    triggered=self.selectRootNodes,
+                                    icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('rootNodes')))
+        self.addAction(selectRoots)
+        self.insertSeparator(selectRoots)
+
+        selectIsolated = QtGui.QAction('Select Isolated Nodes',
+                                       self,
+                                       statusTip="Select nodes with no incoming or outgoing links.",
+                                       triggered=self.selectIsolatedNodes,
+                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('isolatedNodes')))
+        self.addAction(selectIsolated)
+        self.insertSeparator(selectIsolated)
+
+        selectNonIsolated = QtGui.QAction('Select Non Isolated Nodes',
+                                          self,
+                                          statusTip="Select nodes with at least one incoming or outgoing link.",
+                                          triggered=self.selectNonIsolatedNodes,
+                                          icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('nonIsolatedNodes')))
+        self.addAction(selectNonIsolated)
+        self.insertSeparator(selectNonIsolated)
+
+        shortestPath = QtGui.QAction('Find Shortest Path',
+                                     self,
+                                     statusTip="Find the shortest path (if any) between two selected entities.",
+                                     triggered=self.findShortestPath,
+                                     icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('shortestPath')))
+        self.addAction(shortestPath)
+        self.insertSeparator(shortestPath)
+        mergeEntities = QtGui.QAction('Merge Entities',
+                                      self,
+                                      statusTip="Merge the selected entities into one.",
+                                      triggered=self.mergeEntities,
+                                      icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('merge')))
+        self.addAction(mergeEntities)
+        self.insertSeparator(mergeEntities)
+
+        splitEntity = QtGui.QAction('Split Entity',
+                                    self,
+                                    statusTip="Split the selected entity into multiple different ones.",
+                                    triggered=self.splitEntity,
+                                    icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('split')))
+        self.addAction(splitEntity)
+        self.insertSeparator(splitEntity)
+
+        generateReport = QtGui.QAction('Generate Report',
+                                       self,
+                                       statusTip="Generate Report of selected entities.",
+                                       triggered=self.generateReports,
+                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
+        self.addAction(generateReport)
+        self.insertSeparator(generateReport)
+
+        rearrangeCanvas = QtGui.QAction('Rearrange Canvas',
+                                        self,
+                                        statusTip="Rearrange Canvas Nodes automatically.",
+                                        triggered=self.rearrangeGraph,
+                                        icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('rearrange')))
+        self.addAction(rearrangeCanvas)
+        self.insertSeparator(rearrangeCanvas)
+
+        self.linkItems = []
+        self.selectMode = False
+
+    def createManualLink(self):
+        self.parent().toggleLinkingMode()
+
+    def rearrangeGraph(self):
+        self.parent().centralWidget().tabbedPane.getCurrentScene().rearrangeGraph()
+
+    def addCanvas(self):
+        self.parent().addCanvas()
+
+    def configureProxySettings(self):
+        pass
+
+    ###
+    def publishCanvas(self):
+        pass
+
+    ###
+    def selectLeafNodes(self):
+        self.parent().selectLeafNodes()
+
+    def selectRootNodes(self):
+        self.parent().selectRootNodes()
+
+    def selectIsolatedNodes(self):
+        self.parent().selectIsolatedNodes()
+
+    def selectNonIsolatedNodes(self):
+        self.parent().selectNonIsolatedNodes()
+
+    def findShortestPath(self):
+        self.parent().findShortestPath()
+
+    def uploadSelectedFiles(self):
+        self.parent().uploadFiles()
+
+    def downloadSelectedFiles(self):
+        self.parent().downloadFile()
+
+    def mergeEntities(self):
+        self.parent().mergeEntities()
+
+    def splitEntity(self):
+        self.parent().splitEntity()
+
+    def generateReports(self):
+        self.parent().generateReport()

Core/MessageHandler.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+import logging
+from logging import handlers
+
+from multiprocessing import Queue
+from pathlib import Path
+from PySide6 import QtWidgets
+
+
+class MessageHandler:
+    """
+    Class that handles logging.
+    Requires a parent object which implements a 'getSettings' function
+      that returns a Settings object.
+    Call 'debug', 'info', 'warning', 'error' or 'critical' depending
+      on severity.
+    """
+
+    def debug(self, message, exc_info=True):
+        logging.debug(message, exc_info=exc_info)
+        return message
+
+    def info(self, message, popUp=False, exc_info=False):
+        logging.info(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.information(msgBox,
+                                              self.mainWindow.tr("Info"),
+                                              self.mainWindow.tr(message))
+        return message
+
+    def warning(self, message, popUp=False, exc_info=False):
+        logging.warning(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.warning(msgBox,
+                                          self.mainWindow.tr("Warning"),
+                                          self.mainWindow.tr(message))
+        return message
+
+    def error(self, message, popUp=True, exc_info=True):
+        logging.error(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.critical(msgBox,
+                                           self.mainWindow.tr("Error"),
+                                           self.mainWindow.tr(message))
+        return message
+
+    def critical(self, message, popUp=True, exc_info=True):
+        logging.critical(message, exc_info=exc_info)
+        if popUp:
+            msgBox = QtWidgets.QMessageBox()
+            QtWidgets.QMessageBox.critical(msgBox,
+                                           self.mainWindow.tr("Critical"),
+                                           self.mainWindow.tr(message))
+        return message
+
+    # Set the severity level
+    def setSeverityLevel(self, level):
+        currentLogLevel = logging.root.level
+        try:
+            level = int(level)
+            logging.root.setLevel(level)
+            self.mainWindow.SETTINGS.setValue("Logging/Severity", logging.root.level)
+        except ValueError:
+            self.warning("Invalid Severity Level specified.")
+            logging.root.setLevel(currentLogLevel)
+            self.mainWindow.SETTINGS.setValue("Logging/Severity", currentLogLevel)
+
+    def getSeverityLevel(self):
+        return logging.root.level
+
+    def changeLogfile(self, newLogFile):
+        self.mainWindow.SETTINGS.setValue("Logging/Logfile", newLogFile)
+        self.logFileHandler = logging.FileHandler(
+            self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
+        rootLogger = logging.getLogger()
+        for handler in rootLogger.handlers[:]:
+            if isinstance(handler, logging.FileHandler):
+                rootLogger.removeHandler(handler)
+        rootLogger.addHandler(self.logFileHandler)
+
+    def __init__(self, parentObject):
+        self.mainWindow = parentObject
+        self.logQueue = Queue()
+        self.logFileHandler = logging.FileHandler(
+            self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
+        self.logQueueHandler = handlers.QueueHandler(self.logQueue)
+        self.logFormatter = logging.Formatter(
+            '{' + self.mainWindow.SETTINGS.value("Project/Name", "Untitled") + '} [%(asctime)s] - %(levelname)s: %('
+                                                                               'message)s')
+        self.logFormatterQueue = logging.Formatter('[%(asctime)s] - %(levelname)s: %(message)s')
+        self.logFileHandler.setFormatter(self.logFormatter)
+        self.logQueueHandler.setFormatter(self.logFormatterQueue)
+        rootLogger = logging.getLogger()
+        for handler in rootLogger.handlers[:]:
+            if isinstance(handler, logging.FileHandler):
+                rootLogger.removeHandler(handler)
+        rootLogger.addHandler(self.logFileHandler)
+        rootLogger.addHandler(self.logQueueHandler)
+
+        self.setSeverityLevel(self.mainWindow.SETTINGS.value("Logging/Severity", logging.INFO))

Core/PathHelper.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+"""
+This file contains helper functions to validate paths on filesystems.
+Reference: https://stackoverflow.com/a/34102855
+
+Just call 'is_path_exists_or_creatable_portable' with the path (as a string) to validate it.
+"""
+
+import os
+import sys
+import tempfile
+import errno
+
+# Sadly, Python fails to provide the following magic number for us.
+ERROR_INVALID_NAME = 123
+'''
+Windows-specific error code indicating an invalid pathname.
+
+See Also
+----------
+https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
+    Official listing of all such codes.
+'''
+
+
+def is_pathname_valid(pathname: str) -> bool:
+    """
+    `True` if the passed pathname is a valid pathname for the current OS;
+    `False` otherwise.
+    """
+    # If this pathname is either not a string or is but is empty, this pathname
+    # is invalid.
+    try:
+        if not isinstance(pathname, str) or not pathname:
+            return False
+
+        # Strip this pathname's Windows-specific drive specifier (e.g., `C:\`)
+        # if any. Since Windows prohibits path components from containing `:`
+        # characters, failing to strip this `:`-suffixed prefix would
+        # erroneously invalidate all valid absolute Windows pathnames.
+        _, pathname = os.path.splitdrive(pathname)
+
+        # Directory guaranteed to exist. If the current OS is Windows, this is
+        # the drive to which Windows was installed (e.g., the "%HOMEDRIVE%"
+        # environment variable); else, the typical root directory.
+        root_dirname = os.environ.get('HOMEDRIVE', 'C:') \
+            if sys.platform == 'win32' else os.path.sep
+        assert os.path.isdir(root_dirname)  # ...Murphy and her ironclad Law
+
+        # Append a path separator to this directory if needed.
+        root_dirname = root_dirname.rstrip(os.path.sep) + os.path.sep
+
+        # Test whether each path component split from this pathname is valid or
+        # not, ignoring non-existent and non-readable path components.
+        for pathname_part in pathname.split(os.path.sep):
+            try:
+                os.lstat(root_dirname + pathname_part)
+            # If an OS-specific exception is raised, its error code
+            # indicates whether this pathname is valid or not. Unless this
+            # is the case, this exception implies an ignorable kernel or
+            # filesystem complaint (e.g., path not found or inaccessible).
+            #
+            # Only the following exceptions indicate invalid pathnames:
+            #
+            # * Instances of the Windows-specific "WindowsError" class
+            #   defining the "winerror" attribute whose value is
+            #   "ERROR_INVALID_NAME". Under Windows, "winerror" is more
+            #   fine-grained and hence useful than the generic "errno"
+            #   attribute. When a too-long pathname is passed, for example,
+            #   "errno" is "ENOENT" (i.e., no such file or directory) rather
+            #   than "ENAMETOOLONG" (i.e., file name too long).
+            # * Instances of the cross-platform "OSError" class defining the
+            #   generic "errno" attribute whose value is either:
+            #   * Under most POSIX-compatible OSes, "ENAMETOOLONG".
+            #   * Under some edge-case OSes (e.g., SunOS, *BSD), "ERANGE".
+            except OSError as exc:
+                if hasattr(exc, 'winerror'):
+                    if exc.winerror == ERROR_INVALID_NAME:
+                        return False
+                elif exc.errno in {errno.ENAMETOOLONG, errno.ERANGE}:
+                    return False
+    # If a "TypeError" exception was raised, it almost certainly has the
+    # error message "embedded NUL character" indicating an invalid pathname.
+    except TypeError:
+        return False
+    # If no exception was raised, all path components and hence this
+    # pathname itself are valid. (Praise be to the curmudgeonly python.)
+    else:
+        return True
+    # If any other exception was raised, this is an unrelated fatal issue
+    # (e.g., a bug). Permit this exception to unwind the call stack.
+    #
+    # Did we mention this should be shipped with Python already?
+
+
+def is_path_sibling_creatable(pathname: str) -> bool:
+    """
+    `True` if the current user has sufficient permissions to create **siblings**
+    (i.e., arbitrary files in the parent directory) of the passed pathname;
+    `False` otherwise.
+    """
+    # Parent directory of the passed path. If empty, we substitute the current
+    # working directory (CWD) instead.
+    dirname = os.path.dirname(pathname) or os.getcwd()
+
+    try:
+        # For safety, explicitly close and hence delete this temporary file
+        # immediately after creating it in the passed path's parent directory.
+        with tempfile.TemporaryFile(dir=dirname):
+            pass
+        return True
+    # While the exact type of exception raised by the above function depends on
+    # the current version of the Python interpreter, all such types subclass the
+    # following exception superclass.
+    except EnvironmentError:
+        return False
+
+
+def is_path_exists_or_creatable_portable(pathname: str) -> bool:
+    """
+    `True` if the passed pathname is a valid pathname on the current OS _and_
+    either currently exists or is hypothetically creatable in a cross-platform
+    manner optimized for POSIX-unfriendly filesystems; `False` otherwise.
+
+    This function is guaranteed to _never_ raise exceptions.
+    """
+    try:
+        # To prevent "os" module calls from raising undesirable exceptions on
+        # invalid pathnames, is_pathname_valid() is explicitly called first.
+        return is_pathname_valid(pathname) and (
+                os.path.exists(pathname) or is_path_sibling_creatable(pathname))
+    # Report failure on non-fatal filesystem complaints (e.g., connection
+    # timeouts, permissions issues) implying this path to be inaccessible. All
+    # other exceptions are unrelated fatal issues and should not be caught here.
+    except OSError:
+        return False

Core/ReportGeneration.py

@@ -0,0 +1,461 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+from pathlib import Path
+from reportlab.lib import colors
+from reportlab.platypus.doctemplate import PageTemplate, BaseDocTemplate
+from reportlab.graphics.charts.piecharts import Pie
+from reportlab.platypus.frames import Frame
+from reportlab.platypus.tableofcontents import TableOfContents
+from reportlab.lib.units import cm
+
+from reportlab.pdfgen import canvas
+from reportlab.platypus import (Paragraph, PageBreak, Image, Spacer, Table,
+                                ParagraphAndImage)
+from reportlab.lib.enums import TA_LEFT, TA_CENTER, TA_JUSTIFY
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.pagesizes import LETTER, inch
+from reportlab.graphics.shapes import Line, Drawing
+
+
+class MyDocTemplate(BaseDocTemplate):
+    def __init__(self, filename, **kw):
+        self.allowSplitting = 0
+        BaseDocTemplate.__init__(self, filename, **kw)
+        template = PageTemplate('normal', [Frame(3.1 * cm, 2.5 * cm, 15 * cm, 25 * cm, id='F1')])
+        self.addPageTemplates(template)
+
+    def afterFlowable(self, flowable):
+        # Registers TOC entries.
+        if flowable.__class__.__name__ == 'Paragraph':
+            text = flowable.getPlainText()
+            style = flowable.style.name
+            if style == 'Heading1':
+                self.notify('TOCEntry', (0, text, self.page))
+            if style == 'Heading2':
+                self.notify('TOCEntry', (1, text, self.page))
+
+
+class ReportBuilder(canvas.Canvas):
+
+    def __init__(self, *args, **kwargs):
+        canvas.Canvas.__init__(self, *args, **kwargs)
+        self.pages = []
+        self.width, self.height = LETTER
+
+    def showPage(self):
+        self.pages.append(dict(self.__dict__))
+        self._startPage()
+
+    def save(self):
+        pageCount = len(self.pages)
+        for page in self.pages:
+            self.__dict__.update(page)
+            if self._pageNumber > 1:
+                self.drawHeaderAndFooter(pageCount)
+            canvas.Canvas.showPage(self)
+        canvas.Canvas.save(self)
+
+    def drawHeaderAndFooter(self, pageCount):
+        pageCountString = "Page %s of %s" % (self._pageNumber, pageCount)
+        self.saveState()
+        self.setStrokeColorRGB(0, 0, 0)
+        self.setLineWidth(0.5)
+        self.line(66, 78, LETTER[0] - 66, 78)
+        self.setFont('Times-Roman', 10)
+        self.drawString(LETTER[0] - 128, 65, pageCountString)
+        self.restoreState()
+
+
+class PDFReport:
+
+    def titlePage(self, authors, title, subtitle):  # arguments to be added: authors, title, subtitle
+        headline_style = self.styleSheet["Heading5"]
+        headline_style.alignment = TA_CENTER
+        headline_style.fontSize = 36
+        headline_style.leading = 50
+
+        subtitle_style = self.styleSheet["BodyText"]
+        subtitle_style.fontSize = 20
+        subtitle_style.leading = 30
+        subtitle_style.alignment = TA_CENTER
+
+        report_title = Paragraph('Investigation Report', headline_style)
+        self.elements.append(report_title)
+        spacer = Spacer(5, 5)
+        self.elements.append(spacer)
+
+        report_name = Paragraph(title, headline_style)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+        self.elements.append(report_name)
+
+        report_subtitle = Paragraph(subtitle, subtitle_style)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+        self.elements.append(report_subtitle)
+
+        spacer = Spacer(100, 200)
+        self.elements.append(spacer)
+
+        date = datetime.now().strftime('%Y-%m-%d %H:%m:%S')
+        paragraphStyleOptions = ParagraphStyle('Report', fontSize=9, leading=20, justifyBreaks=1, alignment=TA_LEFT,
+                                               justifyLastLine=1)
+        text = f"""Investigation Report<br/>
+        Authors: {authors}<br/>
+        Date: {date}<br/>
+        <br/>
+        """
+        paragraphReportSummary = Paragraph(text, paragraphStyleOptions)
+        self.elements.append(paragraphReportSummary)
+        self.elements.append(PageBreak())
+
+    def summaryPage(self, userSummary, summaryCanvasImage):  # arguments: user summary input
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        paragraphStyleOptions = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_JUSTIFY,
+                                               justifyLastLine=0)
+        text = userSummary
+        paragraphReportSummary = Paragraph(text, paragraphStyleOptions)
+
+        # path to image needed
+        img = Image(summaryCanvasImage)
+        img.preserveAspectRatio = True
+        img.drawHeight = 5.5 * inch
+        img.drawWidth = 6.5 * inch
+        img.hAlign = 'CENTER'
+
+        self.elements.append(paragraphReportSummary)
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+
+        self.elements.append(img)
+        self.elements.append(PageBreak())
+
+    def entityPage(self, userNotes, entityImagePath, appendixDicts, outgoingLinks, incomingLinks, entity,
+                   outgoingNames, incomingNames, title, appendixNumber):
+        # dict with image and notes for appendix
+
+        COLORBLACK = colors.HexColor(0x241F20)
+
+        entity_notes_header = ''
+        entity_notes = ''
+
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        entity_style = self.styleSheet["Title"]
+        entity_style.fontSize = 24
+        entity_style.leading = 20
+        entity_style.alignment = TA_CENTER
+        entity_style.fontName = 'Times-Bold'
+
+        psSubHeaderText = ParagraphStyle('Heading2', fontSize=12, alignment=TA_CENTER, borderWidth=1,
+                                         fontName='Times-Bold')
+        notesParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=0, alignment=TA_JUSTIFY, justifyLastLine=0)
+
+        entityTitle = Paragraph(title, entity_style)
+
+        notes = f"""
+        <font name="Times-Bold" size="12"> User's Notes:</font><br/> {userNotes} 
+
+        """
+
+        text = Paragraph(notes, notesParagraph)
+        if type(entityImagePath) == Drawing:
+            tbl = ReportImageAndParagraph(text, entityImagePath, side='left', xpad=10, ypad=0)
+        elif entityImagePath.endswith('.png') or entityImagePath.endswith('.jpg'):
+            entityImage = Image(entityImagePath, kind='proportional')
+            entityImage.preserveAspectRatio = True
+            entityImage.drawHeight = 2 * inch
+            entityImage.drawWidth = 2 * inch
+            tbl = ReportImageAndParagraph(text, entityImage, side='left', xpad=10, ypad=0)
+
+        links_subHeader = Paragraph("Entity Links", psSubHeaderText)
+
+        entity_summary_subHeader = Paragraph("Entity Summary", psSubHeaderText)
+
+        links_table_style = [('GRID', (0, 0), (-1, -1), 1, COLORBLACK), ('SPAN', (3, 0), (0, 0)),
+                             ('LINEABOVE', (0, 2), (-1, 2), 1, colors.blue),
+                             ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
+                             ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
+
+        # child UID
+        if outgoingLinks:
+            outgoing_data = [
+                ['Outgoing Links'],
+                ['Resolution Name', 'Child Entity', 'Date Created', 'Notes']]
+            index = 0
+            for link in outgoingLinks:
+                linkName = link['Resolution']
+                parentNode = outgoingNames[index]
+                linkNotes = link['Notes']
+                dateCreated = link['Date Created']
+                outgoing_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
+                index += 1
+            outgoing_table = Table(data=outgoing_data, style=links_table_style, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+        else:
+            outgoing_data = [
+                ['No Outgoing Links']]
+            outgoing_table = Table(data=outgoing_data, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+        # parent UID
+        if incomingLinks:
+            incoming_data = [
+                ['Incoming Links'],
+                ['Resolution Name', 'Parent Entity', 'Date Created', 'Notes']]
+            index = 0
+            for link in incomingLinks:
+                linkName = link['Resolution']
+                parentNode = incomingNames[index]
+                linkNotes = link['Notes']
+                dateCreated = link['Date Created']
+                incoming_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
+                index += 1
+            incoming_table = Table(data=incoming_data, style=links_table_style, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+        else:
+            incoming_data = [
+                ['No Incoming Links']]
+            incoming_table = Table(data=incoming_data, hAlign="CENTER")
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+        tableParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_CENTER,
+                                        justifyLastLine=0)
+
+        ParagraphStyle('Heading2', fontSize=11, justifyBreaks=1, alignment=TA_CENTER,
+                       justifyLastLine=1, fontName='Times-Bold', rightIndent=45)
+        notesHeader = ParagraphStyle('Heading2', fontSize=11, justifyBreaks=1, alignment=TA_CENTER,
+                                     justifyLastLine=1, fontName='Times-Bold')
+        entity_data = [
+            ['Entity Fields'],
+            ['Attribute', 'Value']]
+        entity_table_style = [('GRID', (0, 0), (-1, -1), 1, COLORBLACK), ('SPAN', (1, 0), (0, 0)),
+                              ('LINEABOVE', (0, 2), (-1, 2), 1, colors.blue),
+                              ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
+                              ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
+        for key in list(entity):
+            if key != 'uid' and key != 'Icon' and key != 'Notes' and key != 'Date Last Edited':
+                value = entity[key]
+                entity_data.append([Paragraph(key, tableParagraph), Paragraph(value, tableParagraph)])
+            elif key == 'Notes':
+                entity_notes_header = Paragraph("Entity Notes ", notesHeader)
+                entity_notes = Paragraph(entity[key], notesParagraph)
+        entity_table = Table(data=entity_data, style=entity_table_style, hAlign="CENTER")
+
+        appendix_header = Paragraph(f"Appendix {appendixNumber}", notesHeader)
+        text = []
+        images = []
+        imangeNParagraph = []
+        for appendixDict in appendixDicts:
+            if appendixDict['AppendixEntityImage'] == '':
+                text.append(Paragraph(appendixDict['AppendixEntityNotes'], notesParagraph))
+            elif appendixDict['AppendixEntityNotes'] == '' and appendixDict['AppendixEntityImage'] != '':
+                img = Image(Path(appendixDict['AppendixEntityImage']), kind='proportional')
+                # img.preserveAspectRatio=True
+                img.drawHeight = 2 * inch
+                img.drawWidth = 2 * inch
+                img.hAlign = 'LEFT'
+                images.append(img)
+            elif appendixDict['AppendixEntityNotes'] != '' and appendixDict['AppendixEntityImage'] != '':
+                paragraph = appendixDict['AppendixEntityNotes']
+                img = Image(Path(appendixDict['AppendixEntityImage']), kind='proportional')
+                img.drawHeight = 2 * inch
+                img.drawWidth = 2 * inch
+                imangeNParagraph.append(
+                    ReportImageAndParagraph(Paragraph(paragraph), img, side='left', xpad=10, ypad=0))
+
+        self.elements.append(entityTitle)
+        spacer = Spacer(20, 20)
+        self.elements.append(spacer)
+
+        self.elements.append(tbl)
+        # self.elements.append(img)
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+        self.elements.append(links_subHeader)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(incoming_table)
+        spacer = Spacer(25, 25)
+        self.elements.append(spacer)
+        self.elements.append(outgoing_table)
+
+        if not incomingLinks and not outgoingLinks:
+            print('No links to draw link pie graph for: ' + title)
+        else:
+            pieDraw = Drawing()
+            pie = Pie()
+            pie.x = 150
+            pie.y = 65
+            pie.data = [int(len(incomingLinks)), int(len(outgoingLinks))]
+            pie.sideLabels = 1
+            pie.labels = ['Incoming: ' + str(len(incomingLinks)), 'Outgoing: ' + str(len(outgoingLinks))]
+            pie.slices.strokeWidth = 1
+            if int(len(incomingLinks)) > int(len(outgoingLinks)):
+                pie.slices[0].popout = 5
+            else:
+                pie.slices[1].popout = 5
+            pieDraw.add(pie)
+            self.elements.append(pieDraw)
+
+        self.elements.append(entity_summary_subHeader)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(entity_table)
+        self.elements.append(spacer)
+        self.elements.append(entity_notes_header)
+        self.elements.append(entity_notes)
+        self.elements.append(spacer)
+        self.elements.append(appendix_header)
+        spacer = Spacer(20, 20)
+        self.elements.append(spacer)
+        for elementText in text:
+            self.elements.append(elementText)
+            self.elements.append(spacer)
+        for elementImage in images:
+            self.elements.append(elementImage)
+            self.elements.append(spacer)
+        for elementBoth in imangeNParagraph:
+            self.elements.append(elementBoth)
+            self.elements.append(spacer)
+
+        self.elements.append(PageBreak())
+
+    def graphPage(self, timeLineImage):
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
+
+        ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_LEFT,
+                       justifyLastLine=1)
+
+        img = Image(timeLineImage, kind='proportional')
+        img.drawHeight = 1.3 * inch
+        img.drawWidth = 6 * inch
+        img.hAlign = 'LEFT'
+
+        spacer = Spacer(30, 30)
+        self.elements.append(spacer)
+        self.elements.append(img)
+        self.elements.append(PageBreak())
+
+    def nextPagesHeader(self, isSecondPage, header):
+        if isSecondPage:
+            psHeaderText = ParagraphStyle('Heading1', fontSize=14, alignment=TA_LEFT, borderWidth=1)
+            h = header
+            paragraphReportHeader = Paragraph(h, psHeaderText)
+            self.elements.append(paragraphReportHeader)
+
+            spacer = Spacer(10, 10)
+            self.elements.append(spacer)
+
+            d = Drawing(500, 1)
+            line = Line(-15, 0, 430, 0)
+            line.strokeWidth = 2
+            d.add(line)
+            self.elements.append(d)
+
+            spacer = Spacer(10, 1)
+            self.elements.append(spacer)
+
+            d = Drawing(500, 1)
+            line = Line(-15, 0, 430, 0)
+            line.strokeWidth = 0.5
+            d.add(line)
+            self.elements.append(d)
+
+    def __init__(self, path: str, entityListData, outgoingLinks, incomingLinks, entity, summaryCanvasImage,
+                 timelineImage, entityPrimaryField, incomingNames, outgoingNames):
+        self.path = path
+        self.styleSheet = getSampleStyleSheet()
+        self.elements = []
+
+        self.titlePage(authors=entityListData[1].get('Authors'), title=entityListData[1].get('Title'),
+                       subtitle=entityListData[1].get('Subtitle'))
+
+        tocHeaderStyle = ParagraphStyle('Title', fontSize=16, alignment=TA_CENTER, borderWidth=1)
+        tocTitle = Paragraph('Table of Contents', tocHeaderStyle)
+        h1 = ParagraphStyle(name='Heading1',
+                            fontSize=14,
+                            leading=16)
+        h2 = ParagraphStyle(name='Heading2',
+                            fontSize=12,
+                            leading=14, )
+
+        toc = TableOfContents()
+        toc.levelStyles = [h1, h2]
+        self.elements.append(tocTitle)
+        spacer = Spacer(15, 15)
+        self.elements.append(spacer)
+        self.elements.append(toc)
+        self.elements.append(PageBreak())
+
+        head = 'Summary Report'
+        self.nextPagesHeader(True, head)
+
+        self.summaryPage(userSummary=entityListData[2].get('SummaryNotes'), summaryCanvasImage=summaryCanvasImage)
+
+        imagePath = ''
+        for i in range(3, len(entityListData)):
+            if entityListData[i][0].get('EntityImage') != '':
+                imagePath = entityListData[i][0].get('EntityImage')
+
+            head = f'Entity Report: {entityPrimaryField[i - 3]}'
+            self.nextPagesHeader(True, head)
+            self.entityPage(title=entityPrimaryField[i - 3], userNotes=entityListData[i][0].get('EntityNotes'),
+                            entityImagePath=imagePath,
+                            appendixDicts=entityListData[i][1], outgoingLinks=outgoingLinks[i - 3],
+                            incomingLinks=incomingLinks[i - 3],
+                            entity=entity[i - 3], incomingNames=incomingNames[i - 3], outgoingNames=outgoingNames[i - 3],
+                            appendixNumber=i - 3)
+
+        head = 'Graph Report'
+        self.nextPagesHeader(True, head)
+        self.graphPage(timelineImage)
+
+        # Build
+        self.doc = MyDocTemplate(path)
+        self.doc.multiBuild(self.elements, canvasmaker=ReportBuilder)
+
+
+# Subclass ParagraphAndImage to adjust the alignment of the text with the image in the element.
+class ReportImageAndParagraph(ParagraphAndImage):
+
+    def __init__(self, P, I, xpad=3, ypad=3, side='right'):
+        super().__init__(P, I, xpad, ypad, side)
+        self.hI = ''
+        self.wI = ''
+        self._offsets = ''
+
+    def wrap(self, availWidth, availHeight):
+        wI, hI = self.I.wrap(availWidth, availHeight)
+        self.hI = hI
+        self.wI = wI
+        # work out widths array for breaking
+        self.width = availWidth
+        P = self.P
+        style = P.style
+        xpad = self.xpad
+        ypad = self.ypad
+        leading = style.leading
+        leftIndent = style.leftIndent
+        later_widths = availWidth - leftIndent - style.rightIndent
+        intermediate_widths = later_widths - xpad - wI
+        first_line_width = intermediate_widths - style.firstLineIndent
+        P.width = 0
+        nIW = int((hI + ypad) / (leading * 1.0))
+        P.blPara = P.breakLines([first_line_width] + nIW * [intermediate_widths] + [later_widths])
+        if self._side == 'left':
+            self._offsets = [wI + xpad] * (1 + nIW) + [0]
+        # Make the paragraph line up with the top part of the image if its height is less than
+        #   the height of the image.
+        P.height = max(hI, len(P.blPara.lines) * leading)
+        self.height = max(hI, P.height)
+        return self.width, self.height

Core/ResolutionManager.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+import importlib
+import sys
+from os import listdir
+from pathlib import Path
+
+
+class ResolutionManager:
+
+    # Load all resources needed.
+    def __init__(self, mainWindow, messageHandler):
+        self.messageHandler = messageHandler
+        self.mainWindow = mainWindow
+        self.resolutions = {}
+
+    def loadResolutionsFromDir(self, directory: Path):
+        exceptionsCount = 0
+        if self.resolutions.get(directory.stem) is None:
+            self.resolutions[directory.stem] = {}
+        for resolution in listdir(directory):
+            try:
+                if resolution.endswith('.py'):
+                    resolutionName = resolution[:-3]
+                    spec = importlib.util.spec_from_file_location(
+                        resolutionName,
+                        directory / resolution)
+                    module = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(module)
+                    resClass = getattr(module, resolutionName)
+                    resClassInst = resClass()
+                    resNameString = resClassInst.name.replace('_', ' ')
+                    resolutionDesc = resClassInst.description
+                    originTypes = resClassInst.originTypes
+                    resultTypes = resClassInst.resultTypes
+                    resolutionParameters = resClassInst.parameters
+                    self.resolutions[directory.stem][resNameString] = {'name': resNameString,
+                                                                       'description': resolutionDesc,
+                                                                       'originTypes': originTypes,
+                                                                       'resultTypes': resultTypes,
+                                                                       'parameters': resolutionParameters,
+                                                                       'resolution': resClass
+                                                                       }
+                    self.messageHandler.info("Loaded Resolution: " + resNameString)
+            except Exception as e:
+                self.messageHandler.error("Cannot load resolutions from " + str(directory) + "\n Info: " + repr(e))
+                exceptionsCount += 1
+                if exceptionsCount > 5:
+                    self.messageHandler.critical("Failed loading too many modules to proceed.")
+                    sys.exit(5)
+
+    def getResolutionParameters(self, resolutionCategory, resolutionNameString):
+        resolutionsList = self.resolutions.get(resolutionCategory)
+        if resolutionsList is not None and resolutionNameString in resolutionsList:
+            parameters = self.resolutions[resolutionCategory][resolutionNameString]['parameters']
+            return parameters
+        return None
+
+    def getResolutionOriginTypes(self, resolutionNameString):
+        for category in self.resolutions:
+            if resolutionNameString in self.resolutions[category]:
+                originTypes = self.resolutions[category][resolutionNameString]['originTypes']
+                if '*' in originTypes:
+                    originTypes = self.mainWindow.RESOURCEHANDLER.getAllEntities()
+                return originTypes
+        return None
+
+    def loadResolutionsFromServer(self, serverRes):
+        self.resolutions |= serverRes
+
+    def removeServerResolutions(self):
+        try:
+            self.resolutions.pop("Server Resolutions")
+        except KeyError:
+            pass
+
+    def getResolutionCategories(self):
+        result = []
+        for category in self.resolutions:
+            result.append(category)
+        return result
+
+    def getResolutionsForEntityTypesByCategory(self, eTypes):
+        """
+        Gets a set of entity types, and returns a dictionary with all the resolutions that can take all
+        included types as input.
+        If '*' is part of eTypes, get all Resolutions of all categories.
+        """
+        result = {}
+        for category in self.resolutions:
+            result[category] = []
+            for resolution in self.resolutions[category]:
+                originTypes = self.resolutions[category][resolution]['originTypes']
+                if eTypes.issubset(originTypes) or '*' in originTypes:
+                    result[category].append(resolution)
+        return result
+
+    def getResolutionsInCategory(self, category):
+        if category in self.resolutions:
+            return list(self.resolutions[category])
+        return []
+
+    def getAllResolutions(self):
+        categories = self.getResolutionCategories()
+        result = []
+        for category in categories:
+            result += self.getResolutionsInCategory(category)
+        return result
+
+    def executeResolution(self, resolutionName: str, resolutionEntitiesInput: list, parameters: dict,
+                          resolutionUID: str):
+        for category in self.resolutions:
+            for resolution in self.resolutions[category]:
+                if self.resolutions[category][resolution]['name'] == resolutionName:
+                    if category == "Server Resolutions":
+                        self.mainWindow.executeRemoteResolution(resolutionName, resolutionEntitiesInput, parameters,
+                                                                resolutionUID)
+                        # Returning a bool so we know that the resolution is running on the server.
+                        return True
+
+                    resolutionClass = self.resolutions[category][resolution]['resolution']()
+                    result = resolutionClass.resolution(resolutionEntitiesInput, parameters)
+                    return result

Core/Resolutions/Core/ASNToCIDR.py

@@ -0,0 +1,59 @@
+class ASNToCIDR:
+    # A string that is treated as the name of this resolution.
+    name = "Get CIDR from ASN"
+
+    # A string that describes this resolution.
+    description = "ASN to CIDR"
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'Autonomous System'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Network'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois.net import Net
+        from ipwhois.asn import ASNOrigin
+
+        returnResult = []
+
+        for entity in entityJsonList:
+            if entity['Entity Type'] == 'Autonomous System':
+                ipWithPrefix = entity[list(entity)[2]]
+                uid = entity['uid']
+                split_string = ipWithPrefix.split("/", 1)
+                ipWithOutPrefix = split_string[0]
+                ASN = entity["AS Number"]
+                try:
+                    net = Net(ipWithOutPrefix)
+                except Exception:
+                    net = Net('1.1.1.1')
+                obj = ASNOrigin(net)
+                results = obj.lookup(asn=ASN)
+
+                for network in results['nets']:
+                    cidrWithPrefix = network['cidr']
+                    split_string = cidrWithPrefix.split("/", 1)
+                    cidrWithOutPrefix = split_string[0]
+                    prefix = split_string[1]
+                    index_of_child = len(returnResult)
+                    returnResult.append([{'IP Address': cidrWithOutPrefix,
+                                          'Range': prefix,
+                                          'Entity Type': 'Network'},
+                                         {uid: {'Resolution': 'ASN to CIDR', 'Notes': ''}}])
+                    returnResult.append(
+                        [{'Phrase': network['description'], 'Entity Type': 'Phrase'},
+                         {index_of_child: {'Resolution': 'CIDR Description', 'Notes': ''}}])
+                    returnResult.append(
+                        [{'Organization Name': network['source'], 'Entity Type': 'Organization'},
+                         {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}])
+                    returnResult.append(
+                        [{'Company Name': network['maintainer'], 'Entity Type': 'Company'},
+                         {index_of_child: {'Resolution': 'Company Name', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/AffiliateCodesExtractor.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+
+"""
+This resolution can, in rare instances, be a bit unreliable - if it finishes immediately, it's possible that not all
+links were considered, as the page's javascript may not have completely finished loading.
+
+Also, since as far as I can tell one cannot mute the audio of playwright, if a site that autoplays video / audio
+is explored, it is possible that some audio plays while the page is loading and its contents are processed.
+"""
+
+
+class AffiliateCodesExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Affiliate Codes"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of facebook and amazon affiliate codes for websites"
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'},
+                  'Visit External Links': {'description': 'Affiliate codes can be hidden in shortened links, and they '
+                                                          'might not be visible unless one visits the site directly. '
+                                                          'Visiting external links should result in more affiliate '
+                                                          'tags being extracted. Of course, browsing to random pages '
+                                                          'is not always safe.\nVisit external links?',
+                                           'type': 'SingleChoice',
+                                           'value': {'Yes', 'No'}}}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+        import urllib
+        import tldextract
+        import re
+
+        returnResults = []
+        visitExternal = True if parameters['Visit External Links'] == 'Yes' else False
+
+        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        # Sites like youtube replace external links with a redirect link originating
+        #   from the site itself. This sort of gets around that.
+        redirectRegex = re.compile(r'q=[^\s][^&^#]*', re.IGNORECASE)
+
+        amazonRegex = re.compile(r'tag=[^\s][^&]*', re.IGNORECASE)
+        fbRegex = re.compile(r'client_id=\d{4,25}', re.IGNORECASE)
+        webullRegex = re.compile(r'inviteCode=[^\s][^&]*', re.IGNORECASE)
+        smartpassiveincomeRegex = re.compile(r'affcode=[^\s][^&]*', re.IGNORECASE)
+        skillshareRegex = re.compile(r'utm_campaign=[^\s][^&]*', re.IGNORECASE)
+        freetradeRegex = re.compile(r'https://freetrade.app.link/[^\s][^?]*', re.IGNORECASE)
+        freetradeRegex2 = re.compile(r'https://magic.freetrade.io/join/[^\s][^?^#]*', re.IGNORECASE)
+
+        exploredDepth = set()
+        exploredForeign = set()
+        alreadyParsed = set()
+
+        def GetAffiliateCodes(currentUID: str, site: str):
+            requestUrl = str(urllib.parse.unquote(site))
+            if requestUrl not in alreadyParsed:
+                alreadyParsed.add(requestUrl)
+                for affiliateCode in amazonRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Amazon Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in fbRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Facebook Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in webullRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'WeBull Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in smartpassiveincomeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'SmartPassiveIncome Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in skillshareRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'SkillShare Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in freetradeRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'FreeTrade Affiliate Code',
+                                                        'Notes': ''}}])
+                for affiliateCode in freetradeRegex2.findall(requestUrl):
+                    returnResults.append([{'Phrase': affiliateCode,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'FreeTrade Affiliate Code',
+                                                        'Notes': ''}}])
+
+        def extractCodes(currentUID: str, site: str, depth: int):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            linksInLinkHref = soupContents.find_all('link')
+            for tag in linksInLinkHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                            exploredDepth.add(newLink)
+                            extractCodes(currentUID, newLink, newDepth)
+
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink:
+                            redirLinks = redirectRegex.findall(newLink)
+                            if 'redirect' in newLink and len(redirLinks) > 0:
+                                newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
+                                if newLink not in exploredForeign:
+                                    exploredForeign.add(newLink)
+                                    if visitExternal:
+                                        for _ in range(3):
+                                            try:
+                                                page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                                GetAffiliateCodes(currentUID, page.url)
+                                                break
+                                            except TimeoutError:
+                                                pass
+                                else:
+                                    GetAffiliateCodes(currentUID, newLink)
+                            else:
+                                if newLink not in exploredDepth and newDepth > 0:
+                                    exploredDepth.add(newLink)
+                                    extractCodes(currentUID, newLink, newDepth)
+                        elif newLink not in exploredForeign:
+                            exploredForeign.add(newLink)
+                            if visitExternal:
+                                for _ in range(3):
+                                    try:
+                                        page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                        GetAffiliateCodes(currentUID, page.url)
+                                        break
+                                    except TimeoutError:
+                                        pass
+                            else:
+                                GetAffiliateCodes(currentUID, newLink)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+                domain = tldextract.extract(url).fqdn
+                extractCodes(uid, url, maxDepth)
+            browser.close()
+        return returnResults

Core/Resolutions/Core/ContainsPhrase.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+
+class ContainsPhrase:
+    # A string that is treated as the name of this resolution.
+    name = "Contains Phrase"
+
+    # A string that describes this resolution.
+    description = "Checks if a Phrase exists in the Notes of a Phrase entity or the body of a Website."
+
+    originTypes = {'Phrase', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Phrase to Search for': {'description': 'Please enter the Phrase to be searched for.',
+                                           'type': 'String',
+                                           'value': ''},
+                  'Case Sensitive': {'description': 'Do you want the phrase to be case sensitive?',
+                                     'type': 'SingleChoice',
+                                     'value': {'Yes', 'No'}
+                                     }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import re
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            offsets = []
+            counter = 0
+            searchPhrase = parameters['Phrase to Search for']
+            uid = entity['uid']
+            if entity['Entity Type'] == 'Phrase':
+                text = str(entity['Notes'])
+            else:  # Website entity
+                request = requests.get(entity["URL"], headers=headers)
+                text = request.text
+
+            if parameters['Case Sensitive'] == 'No':
+                iterator = re.finditer(rf"{searchPhrase}", text, re.IGNORECASE)
+                for match in iterator:
+                    offsets.append(match.start())
+                    counter += 1
+            elif parameters['Case Sensitive'] == 'Yes':
+                iterator = re.finditer(rf"{searchPhrase}", text)
+                for match in iterator:
+                    offsets.append(match.start())
+                    counter += 1
+
+            returnResults.append([{'Phrase': searchPhrase,
+                                   'Entity Type': 'Phrase',
+                                   'Notes': f'{searchPhrase} was found {counter} times\n'
+                                            f'offsets: Matches at character indices: '
+                                            f'{(", ".join(map(str, offsets)))}'},
+                                  {uid: {'Resolution': 'Contains Phrase',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/DecodePhrase.py

@@ -0,0 +1,55 @@
+class DecodePhrase:
+    # A string that is treated as the name of this resolution.
+    name = "Decode Phrase"
+
+    # A string that describes this resolution.
+    description = "Returns decoded phrase"
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'Phrase'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Phrase'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {'Encoding Type': {'description': 'Select the type of encoding that was applied to the data.',
+                                    'type': 'SingleChoice',
+                                    'value': {'Binary', 'Hexadecimal', 'Base64'}
+                                    }}
+
+    def resolution(self, entityJsonList, parameters):
+        import base64
+        returnResult = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+
+            if parameters['Encoded Type'] == 'Hexadecimal':
+                text = entity[list(entity)[1]].strip()
+                data = str(bytearray.fromhex(text).decode())
+                returnResult.append([{'Phrase': data,
+                                      'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Hexadecimal Phrase', 'Notes': ''}}])
+
+            elif parameters['Encoded Type'] == 'Base64':
+                text = entity[list(entity)[1]].strip()
+                data = base64.b64decode(text).decode()
+                returnResult.append([{'Phrase': str(data),
+                                      'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Base 64 Decoded Phrase', 'Notes': ''}}])
+
+            elif parameters['Encoded Type'] == 'Binary':
+                text = entity[list(entity)[1]].replace(' ', '')
+                if len(text) % 8 != 0:
+                    return "Malformed format not in Octaves"
+
+                ascii_string = ''
+                for binaryIndex in range(0, len(text), 8):
+                    ascii_string += chr(int(text[binaryIndex:binaryIndex + 8], 2))
+                returnResult.append([{'Phrase': str(ascii_string),
+                                      'Entity Type': 'Phrase'},
+                                     {uid: {'Resolution': 'Binary Decoded Phrase', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/EmailExtractor.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+
+class EmailExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Emails"
+
+    # A string that describes this resolution.
+    description = "Returns the email addresses present on a website or index page of a domain."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Email Address'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'},
+                  'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
+                                               'code of the website. However, not all emails on the site may exist in '
+                                               'that format. Using Regex can result in more emails being extracted, '
+                                               'however it is possible that some false positives may be extracted '
+                                               'too.\nDo you want to also use Regex to extract emails, in addition to '
+                                               'the default extraction method?',
+                                'type': 'SingleChoice',
+                                'value': {'Yes', 'No'},
+                                'default': 'Yes'},
+                  'Verify Email Domain Validity': {'description': 'Verification checks are performed on extracted '
+                                                                  'emails to ensure that they are valid and working '
+                                                                  'email addresses. One of these checks involves '
+                                                                  'attempting to resolve the email address domain. '
+                                                                  'This will generate network traffic.\n'
+                                                                  'Do you want to verify email domain validity?',
+                                                   'type': 'SingleChoice',
+                                                   'value': {'Yes', 'No'},
+                                                   'default': 'No'}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+        import tldextract
+        import re
+        from email_validator import validate_email, caching_resolver, EmailNotValidError
+
+        returnResults = []
+
+        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        # Source: https://emailregex.com/
+        # Alt: (?:[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+(\.([a-zA-Z0-9-])+)+)
+        emailRegex = re.compile(r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""")
+        useRegex = True if parameters['Use Regex'] == 'Yes' else False
+
+        resolver = caching_resolver(timeout=10)
+        verifyDomain = True if parameters['Verify Email Domain Validity'] == 'Yes' else False
+
+        exploredDepth = set()
+
+        # The software can deduplicate, but handling it here is better.
+        allEmails = set()
+
+        def extractEmails(currentUID: str, site: str, depth: int):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            if useRegex:
+                potentialEmails = emailRegex.findall(soupContents.get_text())
+                for potentialEmail in potentialEmails:
+                    try:
+                        valid = validate_email(potentialEmail, dns_resolver=resolver, check_deliverability=verifyDomain)
+                        if valid.email not in allEmails:
+                            allEmails.add(valid.email)
+                            returnResults.append([{'Email Address': valid.email,
+                                                   'Entity Type': 'Email Address'},
+                                                  {currentUID: {'Resolution': 'Email Address Found',
+                                                                'Notes': ''}}])
+                    except EmailNotValidError:
+                        pass
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('mailto:'):
+                        try:
+                            valid = validate_email(newLink[7:], dns_resolver=resolver,
+                                                   check_deliverability=verifyDomain)
+                            if valid.email not in allEmails:
+                                allEmails.add(valid.email)
+                                returnResults.append([{'Email Address': valid.email,
+                                                       'Entity Type': 'Email Address'},
+                                                      {currentUID: {'Resolution': 'Email Address Found',
+                                                                    'Notes': ''}}])
+                        except EmailNotValidError:
+                            pass
+                    elif newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                            exploredDepth.add(newLink)
+                            extractEmails(currentUID, newLink, newDepth)
+
+            linksInLinkHref = soupContents.find_all('link')
+            for tag in linksInLinkHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                            exploredDepth.add(newLink)
+                            extractEmails(currentUID, newLink, newDepth)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+                domain = tldextract.extract(url).fqdn
+                extractEmails(uid, url, maxDepth)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/ExtractDOCXMeta.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+
+class ExtractDOCXMeta:
+    # A string that is treated as the name of this resolution.
+    name = "Get DOCX Metadata"
+
+    # A string that describes this resolution.
+    description = "Returns a set of nodes that contain all the metadata info of the file."
+
+    originTypes = {'Document', 'Archive'}
+
+    resultTypes = {'Phrase', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from docx2python import docx2python
+        import magic
+        from pathlib import Path
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+
+            if not (filePath.exists() and filePath.is_file()):
+                continue
+
+            if magic.from_file(str(filePath), mime=True) != \
+                    'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+                continue  # We only care about docx files
+
+            metadata = docx2python(filePath)
+
+            data = metadata.properties
+            defaultDateProperties = ['created', 'modified']
+
+            modifiedDate = str(data.get('modified'))
+            modifiedDate = modifiedDate.replace('Z', '')
+            returnResults.append([{'Date': modifiedDate,
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'modified', 'Notes': ''}}])
+            createdDate = str(data.get('created'))
+            createdDate = createdDate.replace('Z', '')
+            returnResults.append([{'Date': createdDate,
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'created', 'Notes': ''}}])
+
+            for metadataKey in [dataKey for dataKey in data if dataKey not in defaultDateProperties]:
+                returnResults.append([{'Phrase': metadataKey + ': ' + str(data.get(metadataKey)),
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ExtractDateCreated.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+
+class ExtractDateCreated:
+    name = "Extract Date Created"
+    description = "Extract the Date Created field of selected entities"
+    originTypes = {'*'}
+    resultTypes = {'Date'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        return_result = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            return_result.append([{'Date': str(entity['Date Created']),
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Extract Dates', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/ExtractPDFMeta.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+
+class ExtractPDFMeta:
+    # A string that is treated as the name of this resolution.
+    name = "Get PDF Metadata"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of contact info for websites"
+
+    originTypes = {'Document'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from PyPDF2 import PdfFileReader
+        import magic
+        from pathlib import Path
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+
+            if not (filePath.exists() and filePath.is_file()):
+                continue
+
+            if magic.from_file(str(filePath), mime=True) != \
+                    'application/pdf':
+                continue
+
+            with open(filePath, 'rb') as f:
+                pdf = PdfFileReader(f)
+                info = pdf.getDocumentInfo()
+                number_of_pages = pdf.getNumPages()
+
+            for metadataKey in info:
+                if 'Date' in metadataKey:
+                    try:
+                        strDate = info[metadataKey]
+                        strDate = strDate.split(':')[1].split('-')[0]
+                        strDate1 = strDate[:-6]
+                        strDate2 = strDate[-6:]
+                        strDate2 = ':'.join(strDate2[i:i+2] for i in range(0, 6, 2))
+                        strDate1 = strDate1[:-4] + '-' + '-'.join(strDate1[::-1][i:i+2] for i in range(0, 4, 2))[::-1]
+                        strDate = strDate1 + 'T' + strDate2
+                        returnResults.append([{'Date': strDate,
+                                               'Entity Type': 'Date'},
+                                              {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+                    except Exception:
+                        # Reset strDate to default value
+                        strDate = info[metadataKey]
+                        returnResults.append([{'Date': strDate,
+                                               'Entity Type': 'Date'},
+                                              {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+                else:
+                    returnResults.append([{'Phrase': metadataKey + ': ' + str(info[metadataKey]),
+                                           'Entity Type': 'Phrase'},
+                                          {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+
+            returnResults.append([{'Phrase': 'Number of Pages: ' + str(number_of_pages),
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Number of Pages', 'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/FileExtractor.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+
+"""
+It should be noted that the effectiveness of this resolution relies on the website in question to have the non-website
+documents showing their extension in their links in the page. Since the extension is available the vast majority of the
+time, the resolution should produce correct results just about every time.
+
+The alternative is to analyze the contents of each page to determine if they are a non-web file, and then initiate a
+download. This would require a lot of work for very little payoff, which would be error-prone in and of itself, so it
+is not pursued.
+
+A few sites may also present issues when downloads are attempted by a user agent without javascript.
+Even if the download does not work however, the site containing the file will be represented as a node on the graph,
+so the user can download the file themselves if there was any issue.
+"""
+
+
+class FileExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Find Hosted File URLs"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of files in websites and domains."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Website', 'Document', 'Image', 'Video', 'Archive'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        import requests
+        from hashlib import md5
+        from binascii import hexlify
+        from pathlib import Path
+        from bs4 import BeautifulSoup
+        from playwright.sync_api import sync_playwright, TimeoutError
+
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        fileTypes = (".sxw", ".odt", ".ods", ".odg", ".odp", ".docx", ".xlsx", ".pptx", ".ppsx", ".doc", ".xls",
+                     ".ppt", ".pps", ".pdf", ".wpd", ".raw", ".cr2", ".crw", ".indd", ".rdp", ".ica", ".ico", ".txt",
+                     ".text", ".bak", ".log", ".env", ".pub", ".docm", ".xlsm", ".old", ".csv", ".apk", ".sql", ".cfg",
+                     ".key", ".reg", ".yml", ".yaml", ".mail", ".eml", ".mbox", ".mbx", ".url", ".csr", ".config",
+                     ".mdb", ".user", ".adr", ".ini", ".plist", ".conf", ".dat", ".pcf", ".bok", ".properties", ".json",
+                     ".backup", ".sh", ".py", ".md", ".inc")
+        videoTypes = (".mp3", ".mp4")
+        imageTypes = (".jpg", ".jpeg", ".png", ".svg", ".svgz")
+        archiveTypes = (".zip", ".rar", ".7z", ".gz")
+
+        returnResults = []
+
+        def iterateOnDepth(currentURL, currentDepth: int):
+            urlsExplored.add(currentURL)
+            urlsToExplore = set()
+
+            for _ in range(3):
+                try:
+                    page.goto(currentURL, wait_until="networkidle", timeout=10000)
+                    break
+                except TimeoutError:
+                    pass
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            urlInPage = soupContents.find_all('a') + soupContents.find_all('link')
+            for tag in urlInPage:
+                link = tag.get('href', None)
+                if link is not None:
+                    if not link.startswith('http'):
+                        # We assume that we will be redirected to https if available.
+                        link = 'http://' + domain + link
+                    link = link.split('#')[0]
+                    if link not in urlsExplored:
+                        urlsExplored.add(link)
+
+                        fileTypeIdentified = ''
+
+                        # Material name is the part after the last slash of the URL, plus the sha512sum of the URL
+                        if link.endswith(fileTypes):
+                            fileTypeIdentified = 'Document'
+                        elif link.endswith(videoTypes):
+                            fileTypeIdentified = 'Video'
+                        elif link.endswith(imageTypes):
+                            fileTypeIdentified = 'Image'
+                        elif link.endswith(archiveTypes):
+                            fileTypeIdentified = 'Archive'
+
+                        if fileTypeIdentified:
+                            childIndex = len(returnResults)
+
+                            returnResults.append([{'URL': link,
+                                                   'Entity Type': 'Website'},
+                                                  {uid: {'Resolution': 'File URL',
+                                                         'Notes': ''}}])
+
+                            docProperName = link.split('/')[-1]
+                            docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
+                            docFullPath = Path(parameters['Project Files Directory']) / docFileName
+
+                            try:
+                                response = requests.get(link,
+                                                        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+                                                                               'x64; rv:94.0) Gecko/20100101 '
+                                                                               'Firefox/94.0'},
+                                                        stream=True)
+                                with open(docFullPath, 'wb') as fileToWrite:
+                                    for chunk in response.iter_content(4096):
+                                        fileToWrite.write(chunk)
+
+                                returnResults.append([{fileTypeIdentified + ' Name': docProperName,
+                                                       'File Path': docFileName,
+                                                       'Entity Type': fileTypeIdentified},
+                                                      {childIndex: {'Resolution': 'Downloaded File',
+                                                                    'Notes': ''}}])
+                            except Exception:
+                                pass
+
+                        elif domain in link:
+                            urlsToExplore.add(link)
+
+            linksInImgSrc = soupContents.find_all('img')
+            for tag in linksInImgSrc:
+                link = tag.get('src', None)
+                if link is not None:
+                    if not link.startswith('http'):
+                        # We assume that we will be redirected to https if available.
+                        link = 'http://' + domain + link
+                    link = link.split('#')[0]
+                    if link not in urlsExplored:
+                        urlsExplored.add(link)
+
+                        childIndex = len(returnResults)
+
+                        returnResults.append([{'URL': link,
+                                               'Entity Type': 'Website'},
+                                              {uid: {'Resolution': 'File URL',
+                                                     'Notes': ''}}])
+                        docProperName = link.split('/')[-1]
+                        docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
+                        docFullPath = Path(parameters['Project Files Directory']) / docFileName
+
+                        try:
+                            response = requests.get(link,
+                                                    headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+                                                                           'x64; rv:94.0) Gecko/20100101 '
+                                                                           'Firefox/94.0'},
+                                                    stream=True)
+                            with open(docFullPath, 'wb') as fileToWrite:
+                                for chunk in response.iter_content(4096):
+                                    fileToWrite.write(chunk)
+
+                            returnResults.append([{'Image Name': docProperName,
+                                                   'File Path': docFileName,
+                                                   'Entity Type': 'Image'},
+                                                  {childIndex: {'Resolution': 'Downloaded File',
+                                                                'Notes': ''}}])
+                        except Exception:
+                            pass
+
+            if currentDepth > 0:
+                newDepth = currentDepth - 1
+                for newURL in urlsToExplore:
+                    iterateOnDepth(newURL, newDepth)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+                domain = tldextract.extract(url).fqdn
+
+                # Because these do not persist across entities, it is possible to explore a URL multiple times.
+                # However, since different URLs may be encountered at different depths, this way should ensure
+                #   that there are no false negatives, i.e. if something should be discovered, it will be.
+                urlsExplored = set()
+                iterateOnDepth(url, maxDepth)
+
+            return returnResults

Core/Resolutions/Core/FileHasher.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+
+class FileHasher:
+    name = "Get File Hash"
+    description = "Get the Hash of a file"
+    originTypes = {"Image", "Document", "Video", "Archive", "Disk"}
+    resultTypes = {'Hash'}
+    parameters = {'hashing_algorithms': {'description': 'The type of hash/es that will be returned',
+                                         'type': 'MultiChoice',
+                                         'value': {'SHA1', 'SHA256', 'MD5'}
+                                         }}
+
+    def resolution(self, entityJsonList, parameters):
+        import hashlib
+        from pathlib import Path
+
+        return_result = []
+        hashing_algorithms = parameters['hashing_algorithms']
+        for entity in entityJsonList:
+            uid = entity['uid']
+            file_path = Path(entity["File Path"])
+            if not (file_path.exists() and file_path.is_file()):
+                continue
+            block_size = 65536  # The size of each read from the file
+            for hashing_algorithm in hashing_algorithms:
+                if hashing_algorithm == "SHA256":
+                    file_hash = hashlib.sha256()  # nosec
+                elif hashing_algorithm == "SHA1":
+                    file_hash = hashlib.sha1()  # nosec
+                else:
+                    file_hash = hashlib.md5()  # nosec
+                with open(file_path, 'rb') as f:
+                    fb = f.read(block_size)
+                    while len(fb) > 0:
+                        file_hash.update(fb)
+                        fb = f.read(block_size)
+                resulting_hash = file_hash.hexdigest()
+                return_result.append([{'Hash Value': resulting_hash,
+                                       'Hash Algorithm': hashing_algorithm,
+                                       'Entity Type': 'Hash'},
+                                      {uid: {'Resolution': hashing_algorithm + ' Hash', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/GetExternalURLs.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+"""
+This resolution can be notoriously hard to get working. Because one cannot get any indication of when the scripts on
+a page have finished running, it is possible that some results are missed if one is not extremely careful.
+
+Waiting until network traffic has ended should be the safest way to ensure that all javascript has loaded, and as such,
+it would be the most likely point in time where the scripts on the visited page have exposed any useful information
+to be gleamed.
+
+Of course, that runs the risk of the operation timing out for whatever reason, or the operation taking a very long
+time to complete. It also means that making it async risks that some information is missed due to errors, since
+catching them would essentially negate the benefit of doing operations asynchronously.
+
+Tweaks to this script can be made depending on each investigator's workflow.
+"""
+
+
+class GetExternalURLs:
+    name = "Get External Urls"
+    description = "Returns all links to external sites on a website."
+    originTypes = {'Website'}
+    resultTypes = {'Website'}
+
+    parameters = {'Element types to check': {'description': 'Select the types of elements to investigate for '
+                                                            'external links. Note that "a" elements have the lowest '
+                                                            'chance to present false positives, whereas other types '
+                                                            'of elements might point to websites that are affiliated '
+                                                            'with the input URL in some way, such as content delivery '
+                                                            'networks owned by the same entity.',
+                                             'type': 'MultiChoice',
+                                             'value': {'<a> elements', '<img> elements', '<link> elements'},
+                                             'default': ['<a> elements']
+                                             }}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+        import re
+        import urllib
+
+        returnResult = []
+
+        extract_a = '<a> elements' in parameters['Element types to check']
+        extract_img = '<img> elements' in parameters['Element types to check']
+        extract_link = '<link> elements' in parameters['Element types to check']
+
+        # Sites like youtube replace external links with a redirect link originating
+        #   from the site itself. This sort of gets around that.
+        redirectRegex = re.compile(r'q=[^\s][^&^#]*', re.IGNORECASE)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            allPages = []
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+                domain = tldextract.extract(url).fqdn
+
+                # Try to load the page a few times, in case of timeouts.
+                # I don't think making parts of this async actually helps in this case.
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        allPages.append((domain, page, uid))
+                        break
+                    except TimeoutError:
+                        pass
+            for urlVisited in allPages:
+                externalUrls = set()
+                soupContents = BeautifulSoup(urlVisited[1].content(), 'lxml')
+
+                if extract_a:
+                    linksInAHref = soupContents.find_all('a')
+                    for tag in linksInAHref:
+                        link = tag.get('href', None)
+                        if link is not None:
+                            if link.startswith('http'):
+                                if urlVisited[0] not in link:
+                                    externalUrls.add(link.split('#')[0])
+                                else:
+                                    redirLinks = redirectRegex.findall(link)
+                                    if 'redirect' in link and len(redirLinks) > 0:
+                                        newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
+                                        externalUrls.add(newLink)
+
+                if extract_img:
+                    linksInImgSrc = soupContents.find_all('img')
+                    for tag in linksInImgSrc:
+                        link = tag.get('src', None)
+                        if link is not None:
+                            if link.startswith('http'):
+                                if urlVisited[0] not in link:
+                                    externalUrls.add(link.split('#')[0])
+
+                if extract_link:
+                    linksInLinkHref = soupContents.find_all('link')
+                    for tag in linksInLinkHref:
+                        link = tag.get('href', None)
+                        if link is not None:
+                            if link.startswith('http'):
+                                if urlVisited[0] not in link:
+                                    externalUrls.add(link.split('#')[0])
+
+                for externalUrl in externalUrls:
+                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
+                                         {urlVisited[2]: {'Resolution': 'External Link', 'Notes': ''}}])
+            page.close()
+            browser.close()
+
+        return returnResult

Core/Resolutions/Core/GetWebsiteBody.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+
+class GetWebsiteBody:
+    # A string that is treated as the name of this resolution.
+    name = "Get Website Body"
+
+    # A string that describes this resolution.
+    description = "Returns body of websites"
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+
+            primaryField = entity[list(entity)[1]]
+
+            if primaryField.startswith('http://') or primaryField.startswith('https://'):
+                url = primaryField
+            else:
+                url = 'http://' + primaryField
+
+            r = requests.get(url, headers=headers)
+            doc = r.text
+
+            returnResults.append([{'Phrase': 'Website Body: ' + primaryField,
+                                   'Notes': doc,
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Website Body', 'Name': 'Website Body', 'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/HostnameToDomain.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+
+class HostnameToDomain:
+    name = "Hostname to Domain"
+    description = "Find the domain of any given hostname"
+    originTypes = {'Domain', 'Website'}
+    resultTypes = {'Domain'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from tldextract import extract
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]].strip()
+            tsd, td, tsu = extract(primary_field)
+            domain = td + '.' + tsu
+            if domain == primary_field:
+                continue
+            return_result.append([{'Domain Name': domain,
+                                   'Entity Type': 'Domain'},
+                                  {uid: {'Resolution': 'Domain to Hostname', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/HostnameToIP.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+
+class HostnameToIP:
+    name = "Hostname To IP"
+    description = "Gets the IP associated with the given hostname."
+    originTypes = {'Domain', 'Website'}
+    resultTypes = {'IP Address', 'IPv6 Address'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import socket
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            urlFeatures = primaryField.find('://')
+            if urlFeatures != -1:
+                primaryField = primaryField[urlFeatures + 3:]
+            primaryField = primaryField.split('/')[0]  # Remove trailing slashes
+            try:
+                entityResults = socket.getaddrinfo(primaryField, 443, proto=socket.IPPROTO_TCP) + \
+                                socket.getaddrinfo(primaryField, 80, proto=socket.IPPROTO_TCP)
+            except Exception:
+                continue
+
+            for result in entityResults:
+                if result[0].value == 2:
+                    returnResults.append([{'IP Address': result[4][0],
+                                           'Entity Type': 'IP Address'},
+                                          {entity['uid']: {'Resolution': 'Hostname To IP',
+                                                           'Notes': ''}}])
+                elif result[0].value == 10:
+                    returnResults.append([{'IPv6 Address': result[4][0],
+                                           'Entity Type': 'IPv6 Address'},
+                                          {entity['uid']: {'Resolution': 'Hostname To IP',
+                                                           'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/IPToASN.py

@@ -0,0 +1,51 @@
+class IPToASN:
+    # A string that is treated as the name of this resolution.
+    name = "Get ASN From IP"
+
+    # A string that describes this resolution.
+    description = "IP to ASN"
+
+    # A set of entities that this resolution can be ran on.
+    originTypes = {'IP Address'}
+
+    # A set of entities that could be the result of this resolution.
+    resultTypes = {'Autonomous System'}
+
+    # A dictionary of properties for this resolution. The key is the property name,
+    # the value is the property attributes. The type of input expected from the user is determined by the
+    # variable type of the 'value' parameter.
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois.net import Net
+        from ipwhois.asn import IPASN
+        import pycountry
+
+        returnResult = []
+
+        for entity in entityJsonList:
+            if entity['Entity Type'] == 'IP Address':
+                uid = entity['uid']
+                IP_add = entity[list(entity)[1]]
+                net = Net(IP_add)
+                obj = IPASN(net)
+                results = obj.lookup()
+                index_of_child = len(returnResult)
+                countryCode = results['asn_country_code']
+                country = pycountry.countries.get(alpha_2=countryCode).name
+                returnResult.append([{'AS Number': "AS" + results['asn'],
+                                      'ASN Cidr': results['asn_cidr'],
+                                      'Date Created': results['asn_date'],
+                                      'Entity Type': 'Autonomous System'},
+                                     {uid: {'Resolution': 'Autonomous System Results', 'Notes': ''}}])
+                returnResult.append(
+                    [{'Organization Name': results['asn_registry'], 'Entity Type': 'Organization'},
+                     {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}])
+                returnResult.append(
+                    [{'Country Name': country, 'Entity Type': 'Country'},
+                     {index_of_child: {'Resolution': 'Country of Registry for ASN', 'Notes': ''}}])
+                returnResult.append(
+                    [{'Phrase': results['asn_description'], 'Entity Type': 'Phrase'},
+                     {index_of_child: {'Resolution': 'ASN Description', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/IPToHostname.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+
+class IPToHostname:
+    name = "IP To Hostname"
+    description = "Gets the Fully Qualified Domain Name associated with the given IP / IPv6 address."
+    originTypes = {'IPv6 Address', 'IP Address'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import socket
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            try:
+                fqdn = socket.getfqdn(primaryField)
+            except Exception:
+                continue
+
+            if fqdn != primaryField:
+                returnResults.append([{'Domain Name': fqdn,
+                                       'Entity Type': 'Domain'},
+                                      {entity['uid']: {'Resolution': 'Fully Qualified Domain Name',
+                                                       'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/IPWhois.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+
+class IPWhois:
+    name = "IPv4 WhoIs Information"
+    description = "Find information about the Whois of a given IP Address"
+    originTypes = {'IP Address'}
+    resultTypes = {'Country', 'Autonomous System', 'Email Address'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from ipwhois import IPWhois
+        from ipaddress import ip_address
+        import pycountry
+
+        return_result = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]].strip()
+            try:
+                ip_address(primary_field)
+            except ValueError:
+                return "The Entity Provided isn't a valid IP Address"
+            IPobject = IPWhois(primary_field)
+            response = IPobject.lookup_whois()
+            return_result.append([{'AS Number': str(response['asn']),
+                                   'ASN Cidr': str(response['asn_cidr']),
+                                   'Entity Type': 'Autonomous System'},
+                                  {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+            for net in response['nets']:
+                if net['country'] is not None:
+                    country = pycountry.countries.get(alpha_2=net['country']).name
+                    return_result.append([{'Country Name': country,
+                                           'Entity Type': 'Country'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+                if net['name'] is not None:
+                    return_result.append([{'Company Name': net['name'],
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+                if net['emails'] is not None:
+                    for email in net['emails']:
+                        return_result.append([{'Email Address': email,
+                                               'Entity Type': 'Email Address'},
+                                              {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/ImageToDevice.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+
+class ImageToDevice:
+    name = "Device From Image"
+    description = "Find information about on what device the image was taken"
+    originTypes = {"Image"}
+    resultTypes = {'Phrase'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from exif import Image
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            index_of_child = len(return_result)
+            image_path = Path(entity["File Path"])
+            if not (image_path.exists() and image_path.is_file()):
+                continue
+            with open(image_path, 'rb') as image_file:
+                my_image = Image(image_file)
+                if my_image.has_exif is False:
+                    continue
+                else:
+                    for tag in my_image.list_all():
+                        if tag == "make":
+                            return_result.append([{'Phrase': my_image.make,
+                                                   'Entity Type': 'Phrase'},
+                                                  {uid: {'Resolution': 'ExifMetadata Device Manufacturer',
+                                                         'Notes': ''}}])
+                        if tag == "model":
+                            return_result.append([{'Phrase': my_image.model,
+                                                   'Entity Type': 'Phrase'},
+                                                  {index_of_child: {'Resolution': 'ExifMetadata Device Model',
+                                                                    'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/ImageToGeoLocation.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+
+class ImageToGeoLocation:
+    name = "Geolocation From Image"
+    description = "Find information about where the image was taken"
+    originTypes = {"Image"}
+    resultTypes = {'GeoCoordinates'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from exif import Image
+
+        return_result = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            image_path = Path(entity["File Path"])
+            if not (image_path.exists() and image_path.is_file()):
+                continue
+            with open(image_path, 'rb') as image_file:
+                my_image = Image(image_file)
+                if my_image.has_exif is False:
+                    continue
+                else:
+                    for tag in my_image.list_all():
+                        if tag == "gps_latitude":
+                            return_result.append([{'Label': "Location of"+str(entity[list(entity)[1]].strip()),
+                                                   'Latitude': my_image.gps_latitude,
+                                                   'Longitude': my_image.gps_longitude,
+                                                   'Entity Type': 'GeoCoordinates'},
+                                                  {uid: {'Resolution': 'GeoCoordinates', 'Notes': ''}}])
+        return return_result

Core/Resolutions/Core/JSCodeExtractor.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+
+class JSCodeExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Tracking Codes"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of 'ca-pub', 'ua' and 'gtm' tracking codes for websites and/or domains."
+
+    originTypes = {'Website', 'Domain'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright
+        import re
+        returnResults = []
+        requestUrlsParsed = set()
+
+        uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
+        pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
+        gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
+        gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
+        qualtricsRegex = re.compile(r'\bQ_ZID=[a-zA-Z_0-9]*\b', re.IGNORECASE)
+
+        def GetTrackingCodes(pageUid, requestUrl) -> None:
+            if requestUrl not in requestUrlsParsed:
+                requestUrlsParsed.add(requestUrl)
+                for uaCode in uaRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': uaCode,
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google UA Tracking Code',
+                                                     'Notes': ''}}])
+                for pubCode in pubRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': pubCode,
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google AdSense ca-pub Tracking Code',
+                                                     'Notes': ''}}])
+                for gtmCode in gtmRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': gtmCode,
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google GTM Tracking Code',
+                                                     'Notes': ''}}])
+                for gCode in gRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': gCode,
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Google G Tracking Code',
+                                                     'Notes': ''}}])
+                for qCode in qualtricsRegex.findall(requestUrl):
+                    returnResults.append([{'Phrase': qCode[6:],
+                                           'Entity Type': 'Phrase'},
+                                          {pageUid: {'Resolution': 'Qualtrics Tracking Code',
+                                                     'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+
+                # Subscribe to "request" events.
+                page.on("request", lambda request: GetTrackingCodes(uid, request.url))
+                page.goto(url)
+            page.close()
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/PhoneNumbersExtractor.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+"""
+This extracts only phone numbers that are marked as a 'tel:' hyperlink.
+Extraction of other phone numbers is imprecise, as random numbers may be interpreted as valid phone numbers.
+"""
+
+
+class PhoneNumbersExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Phone Numbers"
+
+    # A string that describes this resolution.
+    description = "Returns the Phone Numbers present on a website or index page of a domain."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phone Number'}
+
+    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
+                                               'explored to discover more entities. Each entity discovered after '
+                                               'exploring sites linked in the original website or domain is said to '
+                                               'have a "depth" value of 1. Entities found from exploring the links on '
+                                               'this page would have a "depth" of 2, and so on. A larger value could '
+                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
+                                               'The default value is "0", which means only the provided website, or '
+                                               'the index page of the domain provided, is explored.',
+                                'type': 'String',
+                                'value': '0',
+                                'default': '0'}}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+        import tldextract
+
+        returnResults = []
+
+        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
+        try:
+            maxDepth = max(int(parameters['Max Depth']), 0)
+        except ValueError:
+            return "Invalid value provided for Max Webpages to follow."
+
+        exploredDepth = set()
+
+        def extractTels(currentUID: str, site: str, depth: int):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+            linksInAHref = soupContents.find_all('a')
+            for tag in linksInAHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('tel:'):
+                        returnResults.append([{'Phone Number': newLink[4:],
+                                               'Entity Type': 'Phone Number'},
+                                              {currentUID: {'Resolution': 'Phone Number Found',
+                                                            'Notes': ''}}])
+                    elif newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                            exploredDepth.add(newLink)
+                            extractTels(currentUID, newLink, newDepth)
+
+            linksInLinkHref = soupContents.find_all('link')
+            for tag in linksInLinkHref:
+                newLink = tag.get('href', None)
+                if newLink is not None:
+                    if newLink.startswith('http'):
+                        newLink = newLink.split('#')[0]
+                        newDepth = depth - 1
+                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                            exploredDepth.add(newLink)
+                            extractTels(currentUID, newLink, newDepth)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = 'http://' + url
+                domain = tldextract.extract(url).fqdn
+                extractTels(uid, url, maxDepth)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/PhraseSimilarity.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+class PhraseSimilarity:
+    name = "Perform Similarity Check"
+    description = "Find the ration of similarity between two strings"
+    originTypes = {'*'}
+    resultTypes = {'Phrase'}
+    parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
+                                             'type': 'SingleChoice',
+                                             'value': {'Notes', 'Primary Field'}},
+                  'Algorithm': {'description': 'Select the Algorithm to use',
+                                'type': 'SingleChoice',
+                                'value': {'levenshtein distance', 'damerau levenshtein distance',
+                                          'jaro distance', 'jaro winkler similarity',
+                                          'match rating comparison', 'hamming distance'}}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import jellyfish
+        from itertools import combinations
+
+        return_result = []
+        primary_fields = []
+        notes_fields = []
+        uidList = []
+        selection = parameters['Primary field or Notes']
+        algorithm = parameters['Algorithm'].replace(" ", "_")
+        for entity in entityJsonList:
+            uidList.append(entity['uid'])
+            if selection == 'Primary Field':
+                primary_fields.append(entity[list(entity)[1]].strip())
+            elif selection == 'Notes':
+                notes_fields.append(entity['Notes'])
+
+        if selection == "Notes" and len(notes_fields) > 1:
+            value = list(combinations(notes_fields, 2))
+            uid = list(combinations(uidList, 2))
+        elif selection == "Primary Field" and len(primary_fields) > 1:
+            value = list(combinations(primary_fields, 2))
+            uid = list(combinations(uidList, 2))
+        else:
+            return "Please Select 2 or more entities for comparison"
+        for field in value:
+            similarity = getattr(jellyfish, algorithm)(field[0], field[1])
+            return_result.append([{'Phrase': f"{similarity} similarity",
+                                   'Entity Type': 'Phrase'},
+                                  {uid[value.index(field)][0]: {'Resolution': 'Notes Word Counter', 'Notes': ''}}])
+            return_result.append([{'Phrase': f"{similarity} similarity",
+                                   'Entity Type': 'Phrase'},
+                                  {uid[value.index(field)][1]: {'Resolution': 'Notes Word Counter', 'Notes': ''}}])
+
+        return return_result

Core/Resolutions/Core/RegexMatch.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+
+class RegexMatch:
+    # A string that is treated as the name of this resolution.
+    name = "Get Regex Match"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of contact info for websites"
+
+    originTypes = {'Phrase', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Regex Match': {'description': "Please enter the Regex to be searched for.\n"
+                                                 "if any matches are found to be exactly the same as the entity's "
+                                                 "primary field they will be ignored",
+                                  'type': 'String',
+                                  'value': ''},
+                  'Max Results': {'description': 'Please enter the Maximum number of Results to return',
+                                  'type': 'String',
+                                  'value': '',
+                                  'default': '5'},
+                  'Re Flags': {'description': 'Select Re Flags to be used while compiling the regex',
+                               'type': 'MultiChoice',
+                               'value': {'re.I', 're.M', 're.S'}
+                               }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import requests
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        returnResults = []
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        search_param = parameters['Regex Match']
+        flags = parameters['Re Flags']
+        print(flags)
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if entity['Entity Type'] == 'Phrase':
+                text = str(entity['Notes']) + str(entity['Phrase'])
+            else:  # Website entity
+                r = requests.get(entity["URL"], headers=headers)
+                text = r.text
+
+            if len(flags) == 3:
+                search_re = re.findall(search_param, text, flags=re.I | re.S | re.M)
+            elif len(flags) == 2:
+                search_re = re.findall(search_param, text, flags=flags[0] | flags[1])
+            elif len(flags) == 1:
+                search_re = re.findall(search_param, text, flags=flags[0])
+            else:
+                search_re = re.findall(search_param, text)
+
+            if linkNumbers > len(search_re):
+                linkNumbers = int(len(search_re))
+
+            for i in range(linkNumbers):
+                if search_re[i] == entity[list(entity)[1]]:
+                    continue
+                returnResults.append([{'Phrase': search_re[i],
+                                       'Entity Type': 'Phrase',
+                                       'Notes': ''},
+                                      {uid: {'Resolution': 'Phrase Details',
+                                             'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/ToPhrase.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+
+class ToPhrase:
+    name = "Convert To Phrase"
+    description = "Convert the entity in question to a Phrase entity. Uses only the primary field."
+    originTypes = {'*'}
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]]
+            returnResults.append([{'Phrase': primaryField,
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'To Phrase',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/WordCounter.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+
+class WordCounter:
+    name = "Get Number Of Words"
+    description = "Find the number of words in a given phrase"
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+    parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
+                                             'type': 'SingleChoice',
+                                             'value': {'Notes', 'Primary Field'}}}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+
+        return_result = []
+        selection = parameters['Primary field or Notes']
+        for entity in entityJsonList:
+            uid = entity['uid']
+            if selection == "Notes":
+                total = len(re.findall(r'\w+', entity['Notes'].strip()))
+            else:
+                total = len(re.findall(r'\w+', entity["Phrase"].strip()))
+            return_result.append([{'Phrase': f"{total} words",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Notes Word Count', 'Notes': ''}}])
+
+        return return_result

Core/ResourceHandler.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+
+import networkx as nx
+from defusedxml.ElementTree import parse
+from datetime import datetime
+from os import listdir
+from pathlib import Path
+from uuid import uuid4
+from ast import literal_eval
+from base64 import b64decode
+
+from PySide6.QtGui import QIcon
+from PySide6.QtCore import QByteArray
+
+
+class ResourceHandler:
+
+    def getIcon(self, iconName: str):
+        return self.icons[iconName]
+
+    # Load all resources needed.
+    def __init__(self, mainWindow, messageHandler):
+        self.mainWindow = mainWindow
+        self.messageHandler = messageHandler
+        self.entityCategoryList = {}
+
+        self.icons = {"uploading": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "Uploading.png"),
+                      "uploaded": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                      "Resources" / "Icons" / "Uploaded.png"),
+                      "upArrow": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                     "Resources" / "Icons" / "UpArrow.png"),
+                      "downArrow": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "DownArrow.png"),
+                      "isolatedNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                           "Resources" / "Icons" / "SelectIsolated.png"),
+                      "addCanvas": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "Add_Canvas.png"),
+                      "generateReport": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                            "Resources" / "Icons" / "Generate_Report.png"),
+                      "leafNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "SelectLeaf.png"),
+                      "nonIsolatedNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                              "Resources" / "Icons" / "SelectNonIsolated.png"),
+                      "rootNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "SelectRoot.png"),
+                      "split": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                   "Resources" / "Icons" / "Split.png"),
+                      "merge": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                   "Resources" / "Icons" / "Merge.png"),
+                      "shortestPath": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                          "Resources" / "Icons" / "ShortestPath.png"),
+                      "drawLink": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                      "Resources" / "Icons" / "DrawLink.png"),
+                      "rearrange": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
+                                       "Resources" / "Icons" / "RearrangeGraph.png"),
+                      }
+
+        self.loadCoreEntities()
+
+    def getEntityCategories(self):
+        eList = []
+        for category in self.entityCategoryList:
+            eList.append(category)
+        return eList
+
+    def getAllEntityDetailsWithIconsInCategory(self, category):
+        eList = []
+        for entity in self.entityCategoryList[category]:
+            entityValue = self.entityCategoryList[category][entity]
+            eList.append((self.getBareBonesEntityJson(entity),
+                          entityValue['Icon']
+                          ))
+        return eList
+
+    def getEntityAttributes(self, entityType):
+        aList = []
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        aList.append(attribute)
+                    break
+        except KeyError:
+            self.messageHandler.error("Attempted to get attributes for "
+                                      "nonexistent entity type: " + str(entityType), True)
+            return None
+        return aList
+
+    def getAllEntitiesInCategory(self, category):
+        """
+        Get all Entity Types in the specified category.
+        """
+        eList = []
+        for entity in self.entityCategoryList[category]:
+            eList.append(entity)
+        return eList
+
+    def getAllEntities(self):
+        """
+        Get all recognised Entity Types.
+        """
+        eList = []
+        for category in self.getEntityCategories():
+            for entity in self.getAllEntitiesInCategory(category):
+                eList.append(entity)
+        return eList
+
+    def addRecognisedEntityTypes(self, entityFile) -> bool:
+        try:
+            tree = parse(entityFile, forbid_dtd=True, forbid_entities=True, forbid_external=True)
+        except Exception as exc:
+            self.mainWindow.MESSAGEHANDLER.warning('Error occured when loading entities from '
+                                                   + entityFile + ': ' + str(exc) + ', skipping.')
+            return False
+
+        root = tree.getroot()
+
+        category = root.tag.replace('_', ' ')
+        for entity in list(root):
+            entityName = entity.tag.replace('_', ' ')
+            attributes = entity.find('Attributes').text.strip().split(',')
+            icon = entity.find('Icon')
+            if icon is not None:
+                icon = icon.text.strip()
+            elif icon is None or icon == '':
+                icon = 'Default.svg'
+            if self.entityCategoryList.get(category) is None:
+                self.entityCategoryList[category] = {}
+            self.entityCategoryList[category][entityName] = {
+                'Attributes': attributes,
+                'Icon': str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / icon)}
+        return True
+
+    def loadCoreEntities(self):
+        entDir = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Core" / "Entities"
+        for entFile in listdir(entDir):
+            if entFile.endswith('.xml'):
+                self.addRecognisedEntityTypes(entDir / entFile)
+
+    def loadModuleEntities(self):
+        entDir = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Modules"
+        for module in listdir(entDir):
+            for entFile in listdir(entDir / module):
+                if entFile.endswith('.xml'):
+                    self.addRecognisedEntityTypes(
+                        entDir / module / entFile)
+
+    def getEntityJson(self, entityType: str, jsonData=None):
+        eJson = {'uid': str(uuid4())}
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        eJson[attribute] = str(None)
+                    break
+        except KeyError:
+            self.messageHandler.error("Attempted to get attributes for "
+                                      "malformed entity type: " + str(entityType), True)
+            return None
+        eJson['Entity Type'] = entityType
+        eJson['Date Created'] = None
+        eJson['Date Last Edited'] = None
+        eJson['Notes'] = ""
+        eJson['Icon'] = self.getEntityDefaultPicture(entityType)
+
+        if jsonData is not None:
+            for key in eJson:
+                value = jsonData.get(key)
+                if value is not None and value != '':
+                    eJson[key] = value
+
+        utcNow = datetime.isoformat(datetime.utcnow())
+        if eJson['Date Created'] is None:
+            eJson['Date Created'] = utcNow
+
+        eJson['Date Last Edited'] = utcNow
+
+        return eJson
+
+    def getPrimaryFieldForEntityType(self, entityType: str):
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    return self.entityCategoryList[category][entityType]['Attributes'][0]
+        except KeyError:
+            self.messageHandler.error("Attempted to get primary attribute for "
+                                      "malformed entity type: " + str(entityType), True)
+            return None
+
+    def getBareBonesEntityJson(self, entityType):
+        eJson = {}
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        eJson[attribute] = str(None)
+                    break
+        except KeyError:
+            self.messageHandler.error("Attempted to get attributes for "
+                                      "malformed entity type: " + str(entityType), True)
+            return None
+        eJson['Entity Type'] = entityType
+
+        return eJson
+
+    def getLinkJson(self, jsonData):
+        linkJson = {}
+        try:
+            linkJson['uid'] = jsonData['uid']
+        except KeyError:
+            return None
+
+        utcNow = datetime.isoformat(datetime.utcnow())
+        linkJson['Resolution'] = str(jsonData.get('Resolution'))  # This way, if it is None, it is cast to a string.
+        linkJson['Date Created'] = jsonData.get('Date Created')
+        if linkJson['Date Created'] is None:
+            linkJson['Date Created'] = utcNow
+        linkJson['Date Last Edited'] = utcNow
+        linkJson['Notes'] = str(jsonData.get('Notes'))
+
+        return linkJson
+
+    def getEntityDefaultPicture(self, entityType):
+        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Default.svg"
+        try:
+            for category in self.entityCategoryList:
+                if entityType in self.entityCategoryList[category]:
+                    entityPicture = self.entityCategoryList[category][entityType]['Icon']
+                    if Path(entityPicture).exists():
+                        picture = entityPicture
+                    break
+        except KeyError:
+            self.messageHandler.warning("Attempted to get icon for "
+                                        "nonexistent entity type: " + str(entityType), popUp=False)
+        finally:
+            with open(picture, 'rb') as pictureFile:
+                pictureContents = pictureFile.read()
+            pictureByteArray = QByteArray(pictureContents)
+            return pictureByteArray
+
+    def getLinkPicture(self):
+        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Resolution.png"
+        return QIcon(str(picture)).pixmap(40, 40)
+
+    def getLinkArrowPicture(self):
+        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Right-Arrow.svg"
+        return QIcon(str(picture)).pixmap(40, 40)
+
+    def deconstructGraph(self, graph: nx.DiGraph) -> tuple:
+        nodes = {}
+        for nodeKey in graph.nodes:
+            # Dereference the original dict so we don't actually convert its icon to data.
+            nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
+            try:
+                nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
+            except KeyError:
+                pass
+
+        edges = {edgeKey: graph.edges.get(edgeKey) for edgeKey in graph.edges}
+        return nodes, edges
+
+    def reconstructGraph(self, graphString: str) -> tuple:
+        nodes, edges = literal_eval(graphString)
+        for node in nodes:
+            try:
+                nodes[node]['Icon'] = QByteArray(b64decode(nodes[node]['Icon']))
+            except KeyError:
+                pass
+
+        return nodes, edges

Core/SettingsObject.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+from pickle import dump
+from shutil import move
+from pathlib import Path
+from Core.PathHelper import is_path_exists_or_creatable_portable
+
+
+class SettingsObject(dict):
+    """
+    Stores the following session settings:
+
+    Project/Name: Self explanatory.
+    Project/BaseDir: The directory in which the current project is stored.
+    Program/BaseDir: The directory in which the program is stored.
+    MainWindow/Geometry: Stores the window size
+    MainWindow/WindowState: Saves window state (toolbar position etc.)
+    Logging/Severity: Dictates logging severity
+    Logging/Logfile: Which file should the software write logs to.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.setValue("Project/Name", "Untitled")
+        self.setValue("Program/BaseDir", "Unset")  # dirname(abspath(getsourcefile(lambda:0))) + "/../" )
+        self.setValue("Project/BaseDir", "")
+        self.setValue("Project/FilesDir", "")
+        # For any entity with a Path variable, this dictates whether a copy of the original is made or whether a
+        #   symlink is created. Symlinks require special permissions or developer mode in Windows however.
+        # To ensure that the software works out-of-the-box on all platforms, the default is set to 'Copy'.
+        self.setValue("Project/Symlink or Copy Materials", "Copy")  # Values are 'Copy' or 'Symlink'.
+        self.setValue("Project/Resolution Result Grouping Threshold", "15")
+        self.setValue("Project/Number of Answers Returned", "3")
+        self.setValue("Project/Question Answering Retriever Value", "10")
+        self.setValue("Project/Question Answering Reader Value", "10")
+        self.setValue("Project/Server/Project", "")
+
+        # The value '20' equates to logging.INFO
+        # It's not necessary to set this, but we will for
+        #   the sake of completeness
+        self.setValue("Logging/Severity", "20")
+        self.setValue("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log'))
+
+    # Usability Alias
+    def setValue(self, key, value):
+        self[key] = value
+
+    def value(self, key, alt=None):
+        return self.get(key, alt)
+
+    def save(self):
+        # Save and then move to prevent corruption if the application closes unexpectedly.
+        actualSavePath = str(Path(self.value("Project/BaseDir")).joinpath(self.value("Project/Name") + ".linkscope"))
+        if is_path_exists_or_creatable_portable(actualSavePath):
+            tempSavePath = actualSavePath + '.tmp'
+            projectFile = open(tempSavePath, "wb")
+            dump(self, projectFile)
+            projectFile.close()
+            move(tempSavePath, actualSavePath)

Core/URLManager.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+import zipfile
+import magic
+from pathlib import Path
+from os import symlink
+from shutil import copy2
+from hashlib import md5
+from binascii import hexlify
+
+from PySide6 import QtCore
+
+
+class URLManager:
+    """
+    This class handles URLs. If they are local, the appropriate entity
+    is created to represent the type of file (Document, Image etc).
+
+    The only archive type supported as of now is Zip.
+
+    Note that there is no need to add entities to the database here, as
+    they will be added by the class that is calling functionality from this one.
+    """
+
+    def __init__(self, mainWindow):
+        self.mainWindow = mainWindow
+
+    def handleURLs(self, urls):
+        """
+        Takes a list of QUrls and returns a list of entities that correspond
+        to them.
+        """
+        if len(urls) == 1:
+            url = urls[0]
+            entityJson = self.handleURL(url)
+            return [entityJson]
+
+        returnValue = []
+        for url in urls:
+            returnValue += [self.handleURL(url)]
+
+        return returnValue
+
+    def handleURL(self, url):
+        if not url.isValid():
+            return None
+        if url.isLocalFile():
+            return self.handleLocalURL(url)
+        else:
+            return self.handleRemoteURL(url)
+
+    def handleURLString(self, urlString):
+        self.handleURL(QtCore.QUrl(urlString))
+
+    def handleLocalURL(self, url):
+        urlPath = Path(url.toLocalFile())
+
+        urlName = urlPath.name
+        urlPathString = str(urlPath)
+        savePathString = str(self.moveURLToProjectFilesHelperIfNeeded(urlPath))
+
+        if savePathString == 'None':
+            return None
+
+        # Only support zip files for archives (for now) 10/Jul/2021).
+        if zipfile.is_zipfile(urlPathString):
+            entityJson = {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
+        else:
+            fileType = magic.from_file(urlPathString, mime=True).split('/')[0]
+            if fileType == "video":
+                entityJson = {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
+                pass
+            elif fileType == "image":
+                entityJson = {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
+                pass
+            else:
+                entityJson = {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
+        return entityJson
+
+    def moveURLToProjectFilesHelperIfNeeded(self, urlPath: Path):
+        valuePath = Path(urlPath).absolute()
+
+        if not valuePath.exists() or not valuePath.is_file():
+            return None
+
+        projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).absolute()
+        try:
+            savePath = valuePath.relative_to(projectFilesPath)
+        except ValueError:
+            # The file selected is not in Project Files
+            createSymlink = True if self.mainWindow.SETTINGS.value("Project/Symlink or Copy Materials") == "Symlink" \
+                else False
+            projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir"))
+            # Create a unique path in Project Files
+            saveHash = hexlify(md5(str(urlPath).encode()).digest()).decode()[:8]
+            savePath = projectFilesPath / (saveHash + '|' + urlPath.name)
+
+            if createSymlink:
+                symlink(urlPath, savePath)
+            else:
+                copy2(urlPath, savePath)
+
+            savePath = savePath.relative_to(projectFilesPath)
+
+        return savePath
+
+    def handleRemoteURL(self, url):
+        entity = {'Entity Type': 'Website', 'URL': url.toString()}
+        return entity

LinuxInstaller.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Spec for .desktop files:
+# https://specifications.freedesktop.org/menu-spec/latest/apcs02.html
+# https://specifications.freedesktop.org/menu-spec/latest/index.html
+
+read -r -d '' HELPTEXT << EOF
+== Installation script for LinkScope Client for Ubuntu 20.04 or greater ==
+
+Usage:
+------
+bash $0 install
+- or -
+bash $0 uninstall
+
+Instructions:
+-------------
+Run the script as a user with sudo permissions, with either 'install' or 'uninstall' (no quotes) as the one and only parameter.
+The script will then install or uninstall the software for all users on the computer.
+
+Function:
+---------
+The script will update the package repositories, and attempt to install the following packages:
+p7zip-full curl libopengl0 graphviz libmagic1
+
+Then, the script will download the latest version of the software from AccentuSoft's repository, add it to /usr/local/sbin, and optionally create a Desktop shortcut for the software.
+EOF
+
+read -r -d '' DESKTOP_ENTRY << EOF
+[Desktop Entry]
+Name=LinkScope Client
+StartupWMClass=LinkScope Client
+Comment=Start LinkScope Client
+GenericName=Investigation Software
+Terminal=false
+Exec=/usr/local/sbin/LinkScope/LinkScope
+Icon=/usr/local/sbin/LinkScope/Icon.ico
+Type=Application
+Categories=Application;Office;DataVisualization;
+MimeType=application/linkscope
+Keywords=LinkScope;Investigation;Graph;Knowledge
+EOF
+
+
+# Make sure we're not running as root.
+if [ "$EUID" -eq 0 ]; then
+  echo "Please run the installation script as a user with sudo permissions, not as root."
+  exit
+fi
+
+if [ "$#" -ne 1 ]; then
+  echo "$HELPTEXT"
+  exit
+fi
+
+echo "Sudo permission check"
+sudo echo "Sudo permissions available"
+
+if [ "$?" -ne 0 ]; then
+  echo "Sudo permissions not available, aborting."
+  exit
+fi
+
+if [ "$1" == "install" ]; then
+  echo "Installing Software"
+  echo "Updating Packages"
+  sudo apt update
+  echo "Installing New Packages"
+  sudo apt install p7zip-full curl libopengl0 graphviz libmagic1 -y
+  echo "Downloading latest version of LinkScope client..."
+  linuxURL=$(curl -sL https://github.com/AccentuSoft/LinkScope_Client/releases/latest | grep 'Ubuntu-x64.7z' -m 1 | cut -d '"' -f 2 | tr -d ' ')
+  curl -L https://github.com${linuxURL} -o /tmp/LinkScope.7z
+  sudo 7z x /tmp/LinkScope.7z -o/usr/local/sbin/ && rm /tmp/LinkScope.7z
+  if [ $? -ne 0 ]; then
+    echo "Something went wrong during the download or extraction."
+    echo "Please check that /tmp/LinkScope.7z exists, and that it is an archive containing the latest version of the LinkScope Client software."
+    exit
+  fi
+  sudo echo "$DESKTOP_ENTRY" > /tmp/LinkScope.desktop
+  sudo mv /tmp/LinkScope.desktop /usr/share/applications/LinkScope.desktop
+  sudo chmod +x /usr/share/applications/LinkScope.desktop
+  read -p "Create a Desktop shortcut? WARNING: This will refresh the desktop! [y/N]" -n 1 -r
+  # https://askubuntu.com/a/1014261    -- Making Desktop launchers with .desktop files
+  # https://stackoverflow.com/a/62240478   -- Making .desktop files work.
+  # Not refreshing the desktop would mean that the desktop icon starts working after a restart.
+  # Manually creating a desktop icon through the gui is simpler; after creating a symlink, the user can right-click
+  #   the .desktop file and select 'Allow Launching'. This however requires user interaction.
+  if [[ $REPLY =~ ^[Yy]$ ]]; then
+    ln -s /usr/share/applications/LinkScope.desktop ${HOME}/Desktop/LinkScope.desktop
+    dbus-launch gio set ${HOME}/Desktop/LinkScope.desktop "metadata::trusted" true
+    dbus-send --type=method_call --print-reply --dest=org.gnome.Shell /org/gnome/Shell org.gnome.Shell.Eval string:'global.reexec_self()'
+  fi
+  echo "Done."
+  exit
+fi
+
+if [ "$1" == "uninstall" ]; then
+  echo "Removing Software"
+  sudo rm -rf /usr/local/sbin/LinkScope
+  sudo rm /usr/share/applications/LinkScope.desktop
+  rm ${HOME}/Desktop/LinkScope.desktop
+
+  echo "Done."
+  echo "Note that the system packages used by the software are not removed, in the case that they are necessary for other software to function."
+  exit
+fi
+
+# If the argument passed in is not 'install' or 'uninstall', print the help message.
+echo "$HELPTEXT"

Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByDealer:
+    name = "Aircraft Inquiry By Dealer"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Company"}
+    resultTypes = {'Phrase', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import pandas as pd
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}DealerResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Dealertxt": entity['Company Name']}))
+            for future in as_completed(futures):
+                uid = uidList[futures.index(future)]
+                try:
+                    df_list = pd.read_html(future.result().text)
+                except requests.exceptions.ConnectionError:
+                    return "Please check your internet connection"
+                except ValueError:
+                    return_result = []
+                    return return_result
+                df = df_list[0]
+                for certificate_index in range(len(df["Certificate Number"])):
+                    index_of_child = len(return_result)
+                    return_result.append([{'Company Name': df["Name"][certificate_index],
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'Aircraft Dealer', 'Notes': ''}}])
+                    return_result.append([{'Phrase': df["Certificate Number"][certificate_index],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Aircraft Certificate Number', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByEngine:
+    name = "Aircraft Inquiry By Engine"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Phrase"}
+    resultTypes = {"Phrase"}
+    parameters = {'Manufacturer': {'description': "Enter the Manufacturer of the Engine Model",
+                                   'type': 'String',
+                                   'value': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        Manufacturer = parameters['Manufacturer']
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}EngineReferenceResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Modeltxt": entity['Phrase'],
+                                                               "MfrNametxt": Manufacturer}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            return_result.append([{'Phrase': f"Model Code:str({df['Mfr/Mdl Code']})",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Model Code', 'Notes': ''}}])
+            return_result.append([{'Phrase': f"Engine Type:{df['Type Engine']}",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Engine Type', 'Notes': ''}}])
+            return_result.append([{'Phrase': f"Horse Power:str({df['Horsepower']})",
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Aircraft Engine Horsepower', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByNNumber:
+    name = "Aircraft Inquiry By N-Number"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Phrase"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}NNumberResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"NNumbertxt": entity['Phrase']}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df1 = df_list[0]
+            df2 = df_list[1]
+            df3 = df_list[2]
+            return_result.append([{'ID Number': df1[1][0],
+                                   'Entity Type': 'Identification Number'},
+                                  {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
+            return_result.append([{'Company Name': df1[1][1],
+                                   'Entity Type': 'Company'},
+                                  {uid: {'Resolution': 'Aircraft Company', 'Notes': ''}}])
+            return_result.append([{'Full Name': df2[1][0],
+                                   'Entity Type': 'Person'},
+                                  {uid: {'Resolution': 'Aircraft Owner', 'Notes': ''}}])
+            return_result.append([{'City Name': df2[1][2],
+                                   'Entity Type': 'City'},
+                                  {uid: {'Resolution': "Aircraft Owner's City", 'Notes': ''}}])
+            return_result.append([{'Country Name': df2[1][4],
+                                   'Entity Type': 'Country'},
+                                  {uid: {'Resolution': "Aircraft Owner's Country", 'Notes': ''}}])
+            return_result.append([{'Phrase': df3[1][1],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': "Aircraft Engine Series", 'Notes': ''}}])
+            return_result.append([{'Phrase': df3[1][2],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': "Aircraft Engine Motor", 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryByPersonName:
+    name = "Aircraft Inquiry By Person Name"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Person"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}NameResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"nametxt": entity['Full Name'], "sort_option": "1"}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            for i in range(len(df["N-Number"])):
+                return_result.append([{'Phrase': df["N-Number"][0],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
+                return_result.append([{'ID Number': str(df['Serial Number'][0]),
+                                       'Entity Type': 'Identification Number'},
+                                      {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
+                return_result.append([{'Company Name': df['Manufacturer Name Model'][0],
+                                       'Entity Type': 'Company'},
+                                      {uid: {'Resolution': 'Aircraft Manufacturer Name', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+
+class AircraftInquiryBySerialNumber:
+    name = "AircraftInquiry By Serial Number"
+    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
+    originTypes = {"Identification Number"}
+    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+        import pandas as pd
+
+        futures = []
+        uidList = []
+        return_result = []
+
+        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
+        crafted_url = f"{submit_url}SerialResult"
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                futures.append(session.post(crafted_url, data={"Serialtxt": entity['ID Number'], "sort_option": "1"}))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                df_list = pd.read_html(future.result().text)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            except ValueError:
+                return "No results retrieved"
+            df = df_list[0]
+            for i in range(len(df["N-Number"])):
+                return_result.append([{'Company Name': df["Manufacturer Name"][i],
+                                       'Entity Type': 'Company'},
+                                      {uid: {'Resolution': 'Aircraft Manufacturer', 'Notes': ''}}])
+                return_result.append([{'Phrase': df["N-Number"][i],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
+                return_result.append([{'Phrase': df["Model"][i],
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aircraft Model', 'Notes': ''}}])
+        return return_result

Modules/Aircraft_Inquiry/requirements.txt

@@ -0,0 +1,2 @@
+requests
+pandas

Modules/Aleph/Aleph.xml

@@ -0,0 +1,12 @@
+<Aleph>
+  <Aleph_ID>
+    <Attributes>
+      ID
+    </Attributes>
+  </Aleph_ID>
+  <Aleph_Collection_ID>
+    <Attributes>
+      ID
+    </Attributes>
+  </Aleph_Collection_ID>
+</Aleph>

Modules/Aleph/Aleph_Entity_Search.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+
+
+class Aleph_Entity_Search:
+    name = "Aleph Entity Search"
+    description = "Find information about a given search parameter"
+    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
+    resultTypes = {'Phrase'}
+    parameters = {'Number of results': {'description': 'Creating a lot of nodes could slow down the software. Please '
+                                                       'be mindful of the value you enter.',
+                                        'type': 'String',
+                                        'value': 'Enter the number of results you want returned',
+                                        'default': '1'},
+                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import pycountry
+        import time
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        return_result = []
+        uidList = []
+        futures = []
+
+        url = "https://aleph.occrp.org/api/2/entities"
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        gender = "None"
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        try:
+            max_results = int(parameters['Number of results'])
+        except ValueError:
+            return "The value for parameter 'Max Results' is not a valid integer."
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                crafted_url = url + f"?q={primary_field}&filter:schemata=Thing&limit={max_results}"
+                time.sleep(1)
+                futures.append(session.get(crafted_url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            print(response)
+            for schema in response['results']:
+                index_of_child = len(return_result)
+                try:
+                    if schema['schema'] == "Person":
+                        if schema['properties'].get('gender') is not None \
+                                and schema['properties'].get('gender')[0] == "F":
+                            gender = "Female"
+                        elif schema['properties'].get('gender') is not None \
+                                and schema['properties'].get('gender')[0] == "M":
+                            gender = "Male"
+                        if schema['properties'].get('legalForm') is not None:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties'].get('birthDate')),
+                                  'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}",
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Full Name': str(schema['properties']['name'][0]),
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['birthDate'][0]),
+                                  'Notes': schema['links']['self'],
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
+                        if schema['properties'].get('registrationNumber') is not None:
+                            return_result.append(
+                                [{'Registration Number': str(schema['properties']['registrationNumber'][0]),
+                                  'Notes': '',
+                                  'Entity Type': 'Company'},
+                                 {index_of_child: {'Resolution': 'Aleph Registration Number', 'Notes': ''}}])
+                        if schema['properties'].get('country') is not None:
+                            return_result.append(
+                                [{'Country Name': str(
+                                    pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
+                                    'Notes': '',
+                                    'Entity Type': 'Country'},
+                                    {index_of_child: {'Resolution': 'Country of Origin', 'Notes': ''}}])
+                        if schema['properties'].get('addressEntity'):
+                            return_result.append(
+                                [{'Street Address': str(
+                                    schema['properties']['addressEntity'][0]['properties']['full'][0]),
+                                    'Notes': '',
+                                    'Entity Type': 'Address'},
+                                    {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Street Address': str(schema['properties']['address'][0]),
+                                  'Notes': '',
+                                  'Entity Type': 'Address'},
+                                 {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph Collection ID'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                    elif schema['schema'] == "Organization":
+                        return_result.append(
+                            [{'Organization Name': str(schema['properties']['name'][0]),
+                              'Registration Number': str(schema['properties']['registrationNumber'][0]),
+                              'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}\n"
+                                       f"Source URL: {schema['properties']['sourceUrl'][0]}",
+                              'Entity Type': 'Organization'},
+                             {uid: {'Resolution': 'Aleph Organisation Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Country Name': str(
+                                pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
+                                'Notes': '',
+                                'Entity Type': 'Country'},
+                                {index_of_child: {'Resolution': "Aleph Organisation Country", 'Notes': ''}}])
+                        return_result.append(
+                            [{'Street Address': str(schema['properties']['address'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Address'},
+                             {index_of_child: {'Resolution': "Aleph Organisation Address", 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': "Aleph Organisation ID", 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phone Number': str(schema['properties']['phone'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Phone Number'},
+                             {index_of_child: {'Resolution': 'Phone Number', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['classification'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Organisation Classification', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                    elif schema['schema'] == "Pages":
+                        if 'updated_at' in schema:
+                            date_created = schema['updated_at']
+                        else:
+                            date_created = schema['created_at']
+                        doc_name = 'Document: ' + schema['properties']['title'][0]
+                        entity_link = schema['links']['self']
+                        file_link = schema['links']['file']
+                        source_url = schema['properties']['sourceUrl'][0]
+                        return_result.append(
+                            [{'Phrase': doc_name,
+                              'Source': source_url,
+                              'Notes': 'Link to Aleph Entity: ' + entity_link + '\n\n' +
+                                       'Link to document: ' + file_link,
+                              'Entity Type': 'Phrase',
+                              'Date Created': date_created},
+                             {uid: {'Resolution': 'Aleph Document', 'Notes': ''}}])
+                    elif schema['properties']['parent'][0]['schema'] == "Person":
+                        gender = str(schema['properties']['parent'][0]['properties'].get('gender')[0])
+                        if schema['properties']['parent'][0]['properties'].get('legalForm') is not None:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
+                                  'Notes': f"{schema['properties']['parent'][0]['links']['self']}\nLegal Form: "
+                                           f"{schema['properties']['parent'][0]['properties']['legalForm'][0]}",
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
+                        else:
+                            return_result.append(
+                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
+                                  'Gender': gender,
+                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
+                                  'Notes': schema['properties']['parent'][0]['links']['self'],
+                                  'Entity Type': 'Person'},
+                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
+                        if schema['properties']['parent'][0]['properties'].get('registrationNumber') is not None:
+                            return_result.append(
+                                [{'Registration Number': str(
+                                    schema['properties']['parent'][0]['properties']['registrationNumber'][0]),
+                                    'Notes': '',
+                                    'Entity Type': 'Company'},
+                                    {index_of_child: {'Resolution': 'Company Registration Number', 'Notes': ''}}])
+                        if schema['properties']['parent'][0]['properties'].get('country') is not None:
+                            return_result.append(
+                                [{'Country Name': str(
+                                    pycountry.countries.get(
+                                        alpha_2=schema['properties']['parent'][0]['properties']['country'][0]).name),
+                                    'Notes': '',
+                                    'Entity Type': 'Country'},
+                                    {index_of_child: {'Resolution': 'Country', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['properties']['parent'][0]['id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph ID'},
+                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['label']),
+                              'Notes': str(schema['properties']['parent'][0]['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Phrase'},
+                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
+                        index_of_child_of_child = len(return_result)
+                        return_result.append(
+                            [{'Company Name': str(schema['properties']['name'][0]),
+                              'Notes': schema['links']['self'],
+                              'Entity Type': 'Company'},
+                             {index_of_child: {'Resolution': 'Aleph Company Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Street Address': str(schema['properties']['addressEntity'][0]['properties']['full'][0]),
+                              'Notes': '',
+                              'Entity Type': 'Address'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Person Address', 'Notes': ''}}])
+                        for country_code in schema['collection']['countries']:
+                            return_result.append(
+                                [{'Country Name': str(pycountry.countries.get(alpha_2=country_code).name),
+                                  'Notes': '',
+                                  'Entity Type': 'Country'},
+                                 {index_of_child_of_child: {'Resolution': 'Country', 'Notes': ''}}])
+                        return_result.append(
+                            [{'Phrase': str(schema['collection']['label']),
+                              'Notes': str(schema['collection']['summary']),
+                              'Entity Type': 'Phrase'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
+                        return_result.append(
+                            [{'ID': str(schema['collection']['collection_id']),
+                              'Notes': '',
+                              'Entity Type': 'Aleph Collection ID'},
+                             {index_of_child_of_child: {'Resolution': 'Aleph Entity Search', 'Notes': ''}}])
+                except (TypeError, KeyError) as e:
+                    print(repr(e))
+                    continue
+        return return_result

Modules/Aleph/GetCollectionByID.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+
+class GetCollectionByID:
+    name = "Get Collections By ID"
+    description = "Find information about Collections and their IDs"
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase. Person, Address, Phone Number, Email Address, Country, Bank Account'}
+    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                url = f"https://aleph.occrp.org/api/2/collections/{primary_field}"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            if response['statistics']['names'].get('values') is not None:
+                nameKeys = list(response['statistics']['names'].get('values').keys())
+                for nameKey in nameKeys:
+                    returnResults.append([{'Full Name': str(nameKey),
+                                           'Entity Type': 'Person'},
+                                          {uid: {'Resolution': 'Person Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['addresses'].get('values') is not None:
+                addressKeys = list(response['statistics']['addresses'].get('values').keys())
+                for addressKey in addressKeys:
+                    returnResults.append([{'Street Address': str(addressKey),
+                                           'Entity Type': 'Address'},
+                                          {uid: {'Resolution': 'Address Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['phones'].get('values') is not None:
+                phoneKeys = list(response['statistics']['phones'].get('values').keys())
+                for phoneKey in phoneKeys:
+                    returnResults.append([{'Phone Number': str(phoneKey),
+                                           'Entity Type': 'Phone Number'},
+                                          {uid: {'Resolution': 'Phone Number Entity',
+                                                 'Notes': ''}}])
+
+            if response['statistics']['emails'].get('values') is not None:
+                emailKeys = list(response['statistics']['emails'].get('values').keys())
+                for emailKey in emailKeys:
+                    returnResults.append([{'Email Address': str(emailKey),
+                                           'Entity Type': 'Email Address'},
+                                          {uid: {'Resolution': 'Email Address Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['countries'].get('values') is not None:
+                countriesKeys = list(response['statistics']['countries'].get('values').keys())
+                for countriesKey in countriesKeys:
+                    returnResults.append([{'Country Name': str(countriesKey),
+                                           'Entity Type': 'Country'},
+                                          {uid: {'Resolution': 'Country Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['languages'].get('values') is not None:
+                languagesKeys = list(response['statistics']['languages'].get('values').keys())
+                for languagesKey in languagesKeys:
+                    returnResults.append([{'Phrase': str(languagesKey),
+                                           'Entity Type': 'Phrase'},
+                                          {uid: {'Resolution': 'Language Entity',
+                                                 'Notes': ''}}])
+            if response['statistics']['ibans'].get('values') is not None:
+                ibansKeys = list(response['statistics']['ibans'].get('values').keys())
+                for ibansKey in ibansKeys:
+                    returnResults.append([{'Account Number': str(ibansKey),
+                                           'Entity Type': 'Bank Account'},
+                                          {uid: {'Resolution': 'IBAN Entity',
+                                                 'Notes': ''}}])
+        return returnResults

Modules/Aleph/GetCollectionsInfo.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+
+class GetCollectionsInfo:
+    name = "Get Collections Info"
+    description = "Find information about Collections and their IDs"
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase, Aleph ID'}
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. ',
+                                  'type': 'String',
+                                  'default': '1'},
+                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "The value for parameter 'Max Results' is not a valid integer."
+        if linkNumbers <= 0:
+            return []
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                url = f"https://aleph.occrp.org/api/2/collections?offset=0&limit=300&page"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            max_results = int(len(response['results']))
+            if linkNumbers >= max_results:
+                collections = response['results']
+            else:
+                collections = response['results'][0: linkNumbers]
+
+            for collection in collections:
+                index_of_child = len(returnResults)
+                returnResults.append([{'Phrase': collection['label'],
+                                       'Notes': str(collection.get('summary')),
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': 'Aleph Collection Name',
+                                             'Notes': ''}}])
+
+                returnResults.append([{'ID': collection['id'],
+                                       'Entity Type': 'Aleph ID'},
+                                      {index_of_child: {'Resolution': 'Aleph Collection ID',
+                                                        'Notes': ''}}])
+        return returnResults

Modules/Aleph/GetSimilarEntities.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+
+class GetSimilarEntities:
+    name = "Get Similar Entities"
+    description = "Find information about similar entities"
+    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
+    resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID'}
+    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
+                                                      'It is not intended to amount to advice on which you should place'
+                                                      'sole and entire reliance.\n'
+                                                      'We recommend that you conduct your own independent fact checking'
+                                                      'against the data and materials that you access on Aleph.\n'
+                                                      'Aleph API is not a replacement for traditional due diligence '
+                                                      'checks and know-your-customer background checks.',
+                                       'type': 'String',
+                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
+                                       'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import time
+        import requests
+        import pycountry
+        from requests_futures.sessions import FuturesSession
+        from concurrent.futures import as_completed
+
+        returnResults = []
+        futures = []
+        uidList = []
+
+        if parameters['Aleph Disclaimer'] != 'Accept':
+            return "Please Accept the Terms for Aleph."
+
+        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
+        with FuturesSession(max_workers=15) as session:
+            for entity in entityJsonList:
+                uidList.append(entity['uid'])
+                primary_field = entity[list(entity)[1]].strip()
+                url = f"https://aleph.occrp.org/api/2/entities/{primary_field}/similar"
+                time.sleep(1)
+                futures.append(session.get(url, headers=headers))
+        for future in as_completed(futures):
+            uid = uidList[futures.index(future)]
+            try:
+                response = future.result().json()
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            entities = response['results']
+            for schema in entities:
+                if schema['entity']['schema'] == 'Person':
+                    index_of_child = len(returnResults)
+                    returnResults.append([{'Full Name': ' '.join(map(str, schema['entity']['properties']['name'])),
+                                           'Gender': ' '.join(map(str, schema['entity']['properties']['gender'])),
+                                           'Notes': ' '.join(map(str, schema['entity']['properties']['legalForm'])),
+                                           'Entity Type': 'Person'},
+                                          {uid: {'Resolution': 'Person Entity',
+                                                 'Notes': ''}}])
+                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
+                    returnResults.append([{'Street Address': schema['entity']['properties']['addressEntity'][0]
+                                           ['properties']['full'][0],
+                                           'Postal Code': schema['entity']['properties']['addressEntity'][0]
+                                           ['properties']['postalCode'][0],
+                                           'Country': country,
+                                           'Entity Type': 'Address'},
+                                          {index_of_child: {'Resolution': 'Address',
+                                                            'Notes': ''}}])
+                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
+                                           'Notes': schema['entity']['collection']['summary'],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Location in Database',
+                                                            'Notes': ''}}])
+
+                    returnResults.append([{'ID': schema['entity']['id'],
+                                           'Entity Type': 'Aleph ID'},
+                                          {index_of_child: {'Resolution': 'ID in Database',
+                                                            'Notes': ''}}])
+
+                elif schema['entity']['schema'] == 'Company':
+                    index_of_child = len(returnResults)
+                    returnResults.append([{'Company Name': ' '.join(map(str, schema['entity']['properties']['name'])),
+                                           'Notes': str(schema['entity']['properties']['status']),
+                                           'Entity Type': 'Company'},
+                                          {uid: {'Resolution': 'Company Entity',
+                                                 'Notes': ''}}])
+                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
+                    returnResults.append([{'Street Address': str(schema['entity']['properties'].get('address')),
+                                           'Country': country,
+                                           'Entity Type': 'Address'},
+                                          {index_of_child: {'Resolution': 'Address',
+                                                            'Notes': ''}}])
+                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
+                                           'Notes': schema['entity']['collection']['summary'],
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Location in Database',
+                                                            'Notes': ''}}])
+
+                    returnResults.append([{'ID': schema['entity']['id'],
+                                           'Entity Type': 'Aleph ID'},
+                                          {index_of_child: {'Resolution': 'ID in Database',
+                                                            'Notes': ''}}])
+        return returnResults

Modules/Aleph/requirements.txt

@@ -0,0 +1,2 @@
+requests
+pycountry

Modules/Amass/Amass_Domain.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+# User needs to be in docker group or to have root privileges
+
+class Amass_Domain:
+    name = "Amass Domain Scan"
+    description = "Find information about a particular domain"
+    originTypes = {'Domain'}
+    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain'}
+    parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://virustotal.com.',
+                                 'type': 'String',
+                                 'value': 'None',
+                                 'global': True,
+                                 'default': 'None'},
+                  'AlienVault': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://otx.alienvault.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'BinaryEdge': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://app.binaryedge.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'C99': {'description': 'Enter your api key under your profile after'
+                                         ' signing up on https://c99.nl.',
+                          'type': 'String',
+                          'value': 'None'},
+                  'Censys': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://censys.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Chaos': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://chaos.projectdiscovery.io.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'Cloudflare': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://cloudflare.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'DNSDB': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://dnsdb.info.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'GitHub': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://github.com.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Hunter': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://hunter.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'IPinfo': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://ipinfo.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'NetworksDB': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://networksdb.io.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'PassiveTotal': {'description': 'Enter your api key under your profile after'
+                                                  ' signing up on https://passivetotal.com .',
+                                   'type': 'String',
+                                   'value': 'None'},
+                  'ReconDev': {'description': 'Enter your api key under your profile after'
+                                              ' signing up on https://recon.dev.',
+                               'type': 'String',
+                               'value': 'None'},
+                  'SecurityTrails': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://securitytrails.com.',
+                                     'type': 'String',
+                                     'value': 'None'},
+                  'Shodan': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://shodan.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Spyse': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://spyse.com.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'ThreatBook': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://threatbook.cn.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'Umbrella': {'description': 'Enter your api key under your profile after'
+                                              ' signing up on https://umbrella.cisco.com.',
+                               'type': 'String',
+                               'value': 'None'},
+                  'URLScan': {'description': 'Enter your api key under your profile after'
+                                             ' signing up on https://urlscan.io.',
+                              'type': 'String',
+                              'value': 'None'},
+                  'WhoisXMLAPI': {'description': 'Enter your api key under your profile after'
+                                                 ' signing up on https://whoisxmlapi.com.',
+                                  'type': 'String',
+                                  'value': 'None'},
+                  'ZETAlytics': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://zetalytics.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'ZoomEye': {'description': 'Please Enter the Username and password with a space '
+                                             'in between',
+                              'type': 'String',
+                              'value': 'None'},
+                  'FacebookCT': {'description': 'Please Enter the api key and secret with a space '
+                                                'in between. Obtain them at https://developer.facebook.com',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'Twitter': {'description': 'Please Enter the api key and secret with a space '
+                                             'in between. Obtain them at https://developer.twitter.com',
+                              'type': 'String',
+                              'value': 'None'},
+                  'ReconDev.free': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None'},
+                  'ReconDev.paid': {
+                      'description':
+                          'Please Enter the api key under your profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import json
+        from ipaddress import ip_address, IPv4Address, IPv6Address
+        import docker
+        import tempfile
+        from docker.errors import APIError
+
+        return_result = []
+        client = docker.from_env()
+        # Generate Config as a temporary file:
+        for entity in entityJsonList:
+            primary_field = entity[list(entity)[1]].strip()
+            try:
+                with tempfile.TemporaryDirectory() as tempDir:
+                    tempPath = Path(tempDir).absolute()
+                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                         suffix='Config',
+                                                         dir=tempPath)
+                    config.write("share = true\n")
+                    config.write("[scope]\n")
+                    config.write("port = 80\n")
+                    config.write("port = 443\n")
+                    config.write("[data_sources]\n")
+                    config.write("minimum_ttl = 1440\n")
+                    for parameter in parameters:
+                        if parameters[f'{parameter}'] != 'None':
+                            field1 = f"[data_sources.{parameter}]"
+                            field2 = f"[data_sources.{parameter}.Credentials]"
+                            if parameter == "ZoomEye":
+                                username, password = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"{field2}\n")
+                                config.write(f"username = {username}\n")
+                                config.write(f"password = {password}\n")
+                            elif parameter == "FacebookCT":
+                                field3, secret = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.app1\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                                config.write(f"secret = {secret}\n")
+                            elif parameter == "Twitter":
+                                field3, secret = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.account1\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                                config.write(f"secret = {secret}\n")
+                            elif parameter == "ReconDev.paid":
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.paid\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            elif parameter == "ReconDev.free":
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.free\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            else:
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"{field2}\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            path_to_config = config.name
+                            path_to_config = path_to_config.replace(str(tempPath), "")
+                            config.seek(0)
+                            container = client.containers.run("caffix/amass:latest",
+                                                              f"enum -src -d {primary_field} "
+                                                              f"-config /.config/amass{path_to_config}",
+                                                              volumes={
+                                                                  str(tempPath): {'bind': '/.config/amass',
+                                                                                  'mode': 'rw'}},
+                                                              remove=True)
+                        else:
+                            container = client.containers.run("caffix/amass:latest",
+                                                              f"enum -src -d {primary_field}",
+                                                              volumes={
+                                                                  str(tempPath): {'bind': '/.config/amass',
+                                                                                  'mode': 'rw'}},
+                                                              remove=True)
+                    jsonFile = tempPath / 'amass.json'
+                    if jsonFile.exists():
+                        jsonContents = ""
+                        with open(jsonFile, 'r') as jsonFileHandler:
+                            jsonContents = jsonFileHandler.read()
+            except (APIError, docker.errors.ContainerError) as error:
+                return "Something happened to docker - Cannot continue"
+            uid = entity['uid']
+            for dictionary in jsonContents.splitlines():
+                index_of_child = len(return_result)
+                line_dictionary = json.loads(dictionary)
+                size = len(line_dictionary['addresses'])
+                return_result.append([{'Domain Name': str(line_dictionary['name']),
+                                       'Entity Type': 'Domain'},
+                                      {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
+                for ip in range(size):
+                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
+                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
+                                               'Entity Type': 'IP Address'},
+                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
+                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
+                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
+                                               'Entity Type': 'IPv6 Address'},
+                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
+                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
+                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
+                                           'Entity Type': 'Autonomous System'},
+                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
+                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Amass Domain Scan Description',
+                                                            'Notes': ''}}])
+        jsonFileHandler.close()
+        return return_result

Modules/Amass/Amass_Intel.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# User needs to be in docker group or have root privileges
+
+class Amass_Intel:
+    name = "Amass Intel Scan"
+    description = "Find information about a particular domain"
+    originTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
+    resultTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
+    parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://virustotal.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'AlienVault': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://otx.alienvault.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'BinaryEdge': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://app.binaryedge.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'C99': {'description': 'Enter your api key under your profile after'
+                                         ' signing up on https://c99.nl.',
+                          'type': 'String',
+                          'value': 'None'},
+                  'Censys': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://censys.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Chaos': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://chaos.projectdiscovery.io.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'Cloudflare': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://cloudflare.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'DNSDB': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://dnsdb.info.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'GitHub': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://github.com.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Hunter': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://hunter.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'IPinfo': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://ipinfo.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'NetworksDB': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://networksdb.io.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'PassiveTotal': {'description': 'Enter your api key under your profile after'
+                                                  ' signing up on https://passivetotal.com .',
+                                   'type': 'String',
+                                   'value': 'None'},
+                  'ReconDev': {'description': 'Enter your api key under your profile after'
+                                              ' signing up on https://recon.dev.',
+                               'type': 'String',
+                               'value': 'None'},
+                  'SecurityTrails': {'description': 'Enter your api key under your profile after'
+                                                    ' signing up on https://securitytrails.com.',
+                                     'type': 'String',
+                                     'value': 'None'},
+                  'Shodan': {'description': 'Enter your api key under your profile after'
+                                            ' signing up on https://shodan.io.',
+                             'type': 'String',
+                             'value': 'None'},
+                  'Spyse': {'description': 'Enter your api key under your profile after'
+                                           ' signing up on https://spyse.com.',
+                            'type': 'String',
+                            'value': 'None'},
+                  'ThreatBook': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://threatbook.cn.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'Umbrella': {'description': 'Enter your api key under your profile after'
+                                              ' signing up on https://umbrella.cisco.com.',
+                               'type': 'String',
+                               'value': 'None'},
+                  'URLScan': {'description': 'Enter your api key under your profile after'
+                                             ' signing up on https://urlscan.io.',
+                              'type': 'String',
+                              'value': 'None'},
+                  'WhoisXMLAPI': {'description': 'Enter your api key under your profile after'
+                                                 ' signing up on https://whoisxmlapi.com.',
+                                  'type': 'String',
+                                  'value': 'None'},
+                  'ZETAlytics': {'description': 'Enter your api key under your profile after'
+                                                ' signing up on https://zetalytics.com.',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'ZoomEye': {'description': 'Please Enter the Username and password with a space '
+                                             'in between',
+                              'type': 'String',
+                              'value': 'None'},
+                  'FacebookCT': {'description': 'Please Enter the api key and secret with a space '
+                                                'in between. Obtain them at https://developer.facebook.com',
+                                 'type': 'String',
+                                 'value': 'None'},
+                  'Twitter': {'description': 'Please Enter the api key and secret with a space '
+                                             'in between. Obtain them at https://developer.twitter.com',
+                              'type': 'String',
+                              'value': 'None'},
+                  'ReconDev.free': {
+                      'description': 'Please Enter the api key under your"\
+                       "profile after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None'},
+                  'ReconDev.paid': {
+                      'description': 'Please Enter the api key under your profile"\
+                       "after signing up on https://recon.dev',
+                      'type': 'String',
+                      'value': 'None'}}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import json
+        from ipaddress import ip_address, IPv4Address, IPv6Address
+        import docker
+        import tempfile
+        from docker.errors import APIError
+
+        return_result = []
+        client = docker.from_env()
+        # Generate Config as a temporary file:
+        for entity in entityJsonList:
+            primary_field = entity[list(entity)[1]].strip()
+            try:
+                with tempfile.TemporaryDirectory() as tempDir:
+                    tempPath = Path(tempDir).absolute()
+                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
+                                                         suffix='Config',
+                                                         dir=tempPath)
+                    config.write("share = true\n")
+                    config.write("[scope]\n")
+                    config.write("port = 80\n")
+                    config.write("port = 443\n")
+                    config.write("[data_sources]\n")
+                    config.write("minimum_ttl = 1440\n")
+                    for parameter in parameters:
+                        if parameters[f'{parameter}'] != 'None':
+                            field1 = f"[data_sources.{parameter}]"
+                            field2 = f"[data_sources.{parameter}.Credentials]"
+                            if parameter == "ZoomEye":
+                                username, password = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"{field2}\n")
+                                config.write(f"username = {username}\n")
+                                config.write(f"password = {password}\n")
+                            elif parameter == "FacebookCT":
+                                field3, secret = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.app1\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                                config.write(f"secret = {secret}\n")
+                            elif parameter == "Twitter":
+                                field3, secret = parameters[parameter].split(' ', 1)
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.account1\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                                config.write(f"secret = {secret}\n")
+                            elif parameter == "ReconDev.paid":
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.paid\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            elif parameter == "ReconDev.free":
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"[data_sources.{parameter}.free\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            else:
+                                field3 = parameters[f'{parameter}']
+                                config.write(f"{field1}\n")
+                                config.write(f"{field2}\n")
+                                config.write(f"apikey = \"{field3}\"\n")
+                            path_to_config = config.name
+                            path_to_config = path_to_config.replace(str(tempPath), "")
+                            config.seek(0)
+                            print(config.read())
+                            print(path_to_config)
+                            print(entity['Entity Type'])
+                            if entity['Entity Type'] == "Domain":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -d {primary_field} -config /.config/amass"
+                                                                  f"{path_to_config}",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
+                                    'Entity Type'] == "Company":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -org {primary_field} -whois -config "
+                                                                  f"/.config/amass{path_to_config}",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "IP Address":
+                                try:
+                                    ip_address(primary_field)
+                                except ValueError:
+                                    return "The Entity Provided isn't a valid IP Address"
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -addr {primary_field} -whois -config "
+                                                                  f"/.config/amass{path_to_config}",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "Autonomous System":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}   "
+                                                                  f" -whois -config /.config/amass{path_to_config}",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                        else:
+                            if entity['Entity Type'] == "Domain":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -d {primary_field}",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
+                                    'Entity Type'] == "Company":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -org {primary_field} -whois",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "IP Address":
+                                try:
+                                    ip_address(primary_field)
+                                except ValueError:
+                                    return "The Entity Provided isn't a valid IP Address"
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -addr {primary_field} -whois",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                            elif entity['Entity Type'] == "Autonomous System":
+                                container = client.containers.run("caffix/amass:latest",
+                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}"
+                                                                  f" -whois",
+                                                                  volumes={
+                                                                      str(tempPath): {'bind': '/.config/amass',
+                                                                                      'mode': 'rw'}},
+                                                                  remove=True)
+                    jsonFile = tempPath / 'amass.json'
+                    if jsonFile.exists():
+                        jsonContents = ""
+                        with open(jsonFile, 'r') as jsonFileHandler:
+                            jsonContents = jsonFileHandler.read()
+            except (APIError, docker.errors.ContainerError) as error:
+                return "Soomething happened to docker - Cannot continue."
+            uid = entity['uid']
+            for dictionary in jsonContents.splitlines():
+                index_of_child = len(return_result)
+                line_dictionary = json.loads(dictionary)
+                size = len(line_dictionary['addresses'])
+                return_result.append([{'Domain Name': str(line_dictionary['name']),
+                                       'Entity Type': 'Domain'},
+                                      {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
+                for ip in range(size):
+                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
+                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
+                                               'Entity Type': 'IP Address'},
+                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
+                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
+                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
+                                               'Entity Type': 'IPv6 Address'},
+                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
+                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
+                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
+                                           'Entity Type': 'Autonomous System'},
+                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
+                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
+                                           'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'Amass Intel Scan Description',
+                                                            'Notes': ''}}])
+        jsonFileHandler.close()
+        return return_result

Modules/Amass/requirements.txt

@@ -0,0 +1 @@
+docker

Modules/BigMatch/BigMatch.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+
+class BigMatch:
+    name = "BigMatch Search"
+    description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
+    originTypes = {"Image", "Document", "Archive"}
+    resultTypes = {'Website'}
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+
+        return_result = []
+
+        url = "https://bigmatch.rev.ng/static/index.html"
+        failString = 'Too many strings in binary?'
+        successString = 'Results:'
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+
+            for entity in entityJsonList:
+                uid = entity['uid']
+                file_path = Path(parameters['Project Files Directory']) / entity["File Path"]
+                file_path = file_path.absolute()
+                if not (file_path.exists() and file_path.is_file()):
+                    continue
+                page.wait_for_timeout(3000)
+
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        inputLocator = page.locator("input")
+                        inputLocator.set_input_files([str(file_path)])
+                        page.wait_for_timeout(3000)
+                        soup = BeautifulSoup(page.content(), 'lxml')
+                        soupText = soup.get_text()
+                        while (failString not in soupText) and (successString not in soupText):
+                            page.wait_for_timeout(1000)
+                            soup = BeautifulSoup(page.content(), 'lxml')
+                            soupText = soup.get_text()
+                        if failString in soupText:
+                            return []
+                        for link in soup.find_all('a'):
+                            potentialLink = link.get('href', None)
+                            if potentialLink is not None:
+                                if 'github' in potentialLink:
+                                    return_result.append([{'URL': potentialLink, 'Entity Type': 'Website'},
+                                                          {uid: {'Resolution': 'BigMatch Github Link', 'Notes': ''}}])
+
+                        break
+                    except TimeoutError:
+                        pass
+            page.close()
+            browser.close()
+        return return_result

Modules/BigMatch/requirements.txt

@@ -0,0 +1 @@
+beautifulsoup4

Modules/CryptoCurrency/BlockChainAPI.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+
+class BlockChainAPI:
+    # A string that is treated as the name of this resolution.
+    name = "BlockChainAPI Get Transaction Details"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of balance, sent, received info"
+
+    originTypes = {'Crypto Wallet', 'Phrase'}
+
+    resultTypes = {'Currency'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        import datetime
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            search_address = entity[list(entity)[1]]
+
+            try:
+                getSentByAddress = requests.get(
+                    f'https://blockchain.info/q/getsentbyaddress/{search_address}?confirmations=6')
+                if getSentByAddress.status_code == 404:
+                    continue
+                time.sleep(1)
+                getReceivedByAddress = requests.get(
+                    f'https://blockchain.info/q/getreceivedbyaddress/{search_address}?confirmations=6')
+                time.sleep(1)
+                addressBalance = requests.get(
+                    f'https://blockchain.info/q/addressbalance/{search_address}?confirmations=6')
+                time.sleep(1)
+                addressFirstSeen = requests.get(
+                    f'https://blockchain.info/q/addressfirstseen/{search_address}?confirmations=6')
+                time.sleep(1)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+
+            dateCreated = str(datetime.datetime.fromtimestamp(float(addressFirstSeen.text))).replace(" ", 'T')
+
+            returnResults.append(
+                [{'Amount': str(int(getSentByAddress.text) / 100000000),
+                  'Currency Type': 'BTC',
+                  'Notes': 'sent by address',
+                  'Entity Type': 'Currency'},
+                 {uid: {'Resolution': 'sent by address',
+                        'Name': 'sent by address',
+                        'Notes': ''}}])
+            returnResults.append(
+                [{'Amount': str(int(getReceivedByAddress.text) / 100000000),
+                  'Currency Type': 'BTC',
+                  'Notes': 'received by address',
+                  'Entity Type': 'Currency'},
+                 {uid: {'Resolution': 'received by address',
+                        'Name': 'received by address',
+                        'Notes': ''}}])
+            returnResults.append(
+                [{'Amount': str(int(addressBalance.text) / 100000000),
+                  'Currency Type': 'BTC',
+                  'Notes': 'address balance',
+                  'Entity Type': 'Currency'},
+                 {uid: {'Resolution': 'address balance',
+                        'Name': 'address balance',
+                        'Notes': ''}}])
+            returnResults.append(
+                [{'Date': str(dateCreated),
+                  'Notes': 'first seen date',
+                  'Entity Type': 'Date'},
+                 {uid: {'Resolution': 'first seen date',
+                        'Name': 'first seen date',
+                        'Notes': ''}}])
+            return returnResults

Modules/CryptoCurrency/CryptoCurrency.xml

@@ -0,0 +1,7 @@
+<CryptoCurrency>
+  <Crypto_Wallet>
+    <Attributes>
+      Wallet Address
+    </Attributes>
+  </Crypto_Wallet>
+</CryptoCurrency>

Modules/CryptoCurrency/EtherScanGetBalance.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+
+class EtherScanGetBalance:
+    name = "EtherScan.io Get Balance"
+    description = "EtherScan get the balance of the selected account"
+    originTypes = {"Crypto Wallet"}
+    resultTypes = {'Crypto Wallet'}
+    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
+                                                       "https://etherscan.io",
+                                        'type': 'String',
+                                        'value': '',
+                                        'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+
+        return_result = []
+
+        api_key = parameters['EtherScan API Key']
+        url = "https://api.etherscan.io/api"
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]]
+            crafted_url = f"{url}?module=account&action=balance&address={primary_field}&tag=latest&apikey={api_key}"
+            try:
+                response = requests.get(crafted_url)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            response = response.json()
+            return_result.append([{'Amount': response['result'],
+                                   'Currency': 'Ethereum',
+                                   'Entity Type': 'Currency'},
+                                  {uid: {'Resolution': 'EtherScan.io Account Balance', 'Notes': ''}}])
+        return return_result

Modules/CryptoCurrency/EtherScanGetBlocksMined.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+
+class EtherScanGetBlocksMined:
+    name = "EtherScan.io Get Blocks Mined"
+    description = "EtherScan Blocks Mined from the selected account"
+    originTypes = {"Crypto Wallet"}
+    resultTypes = {'Crypto Wallet'}
+    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
+                                                       "https://etherscan.io",
+                                        'type': 'String',
+                                        'value': '',
+                  'global': True}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+
+        return_result = []
+
+        api_key = parameters['EtherScan API Key']
+        url = "https://api.etherscan.io/api"
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            primary_field = entity[list(entity)[1]]
+            crafted_url =\
+                f"{url}?module=account&action=getminedblocks&address={primary_field}&tag=latest&apikey={api_key}"
+            try:
+                response = requests.get(crafted_url)
+            except requests.exceptions.ConnectionError:
+                return "Please check your internet connection"
+            response = response.json()
+            return_result.append([{'Phrase': response['result'],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'EtherScan.io Blocks Mined', 'Notes': ''}}])
+        return return_result

Modules/CryptoCurrency/requirements.txt

@@ -0,0 +1 @@
+requests

Modules/EDGAR_&_EFD/CompanyInfo.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+class CompanyInfo:
+    # A string that is treated as the name of this resolution.
+    name = "Get Company Info"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes containing Company Information"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Phrase, SIC, EIN, Address'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(1)
+            search_url = f'https://data.sec.gov/submissions/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            time.sleep(1)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+
+            exchanges = data['exchanges']
+            for exchange in exchanges:
+                returnResults.append([{'Exchange Name': exchange,
+                                       'Entity Type': 'Exchanges'},
+                                      {uid: {'Resolution': 'Exchange',
+                                             'Name': 'Exchange',
+                                             'Notes': ''}}])
+
+            tickers = data['tickers']
+            for ticker in tickers:
+                returnResults.append([{'Ticker Name': ticker,
+                                       'Entity Type': 'Tickers'},
+                                      {uid: {'Resolution': 'Exchange',
+                                             'Name': 'Exchange',
+                                             'Notes': ''}}])
+
+            if data['insiderTransactionForOwnerExists'] == 1:
+                returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists',
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+            else:
+                returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists',
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+
+            if data['insiderTransactionForIssuerExists'] == 1:
+                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists',
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+            else:
+                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists',
+                                       'Entity Type': 'Phrase'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+
+            if data['sic'] is not None:
+                returnResults.append([{'SIC': str(data['sic']),
+                                       'Description': data['sicDescription'],
+                                       'Entity Type': 'SIC'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+            if data['ein'] is not None:
+                returnResults.append([{'EIN': str(data['ein']),
+                                       'Entity Type': 'EIN'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+            if data['addresses'] is not None:
+                returnResults.append([{'Street Address': data['addresses']['mailing']['street1'],
+                                       'Postal Code': data['addresses']['mailing']['zipCode'],
+                                       'Country': data['addresses']['mailing']['stateOrCountry'],
+                                       'Locality': data['addresses']['mailing']['city'],
+                                       'Entity Type': 'Address'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+
+            if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']:
+                returnResults.append([{'Street Address': data['addresses']['business']['street1'],
+                                       'Postal Code': data['addresses']['business']['zipCode'],
+                                       'Country': data['addresses']['business']['stateOrCountry'],
+                                       'Locality': data['addresses']['business']['city'],
+                                       'Entity Type': 'Address'},
+                                      {uid: {'Resolution': '',
+                                             'Name': '',
+                                             'Notes': ''}}])
+        return returnResults

Modules/EDGAR_&_EFD/CompanyToCIK.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+
+class CompanyToCIK:
+    # A string that is treated as the name of this resolution.
+    name = "Get CIK ID From Company"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes of contact info for websites"
+
+    originTypes = {'Phrase', 'Company'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        from bs4 import BeautifulSoup
+        from playwright.sync_api import sync_playwright, TimeoutError
+
+        returnResults = []
+        index_of_child = []
+        cikRegex = re.compile(r'CIK=\d{4,10}', re.IGNORECASE)
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+            for entity in entityJsonList:
+                page.wait_for_timeout(1000)
+                uid = entity['uid']
+                search_term = entity[list(entity)[1]]
+                pageResolved = False
+                for _ in range(3):
+                    try:
+                        page.goto(f'https://www.sec.gov/cgi-bin/browse-edgar?company={search_term}',
+                                  wait_until="networkidle", timeout=10000)
+                        pageResolved = True
+                        break
+                    except TimeoutError:
+                        pass
+                if not pageResolved:
+                    continue
+
+                soup = BeautifulSoup(page.content(), 'lxml')
+                cikIDs = cikRegex.findall(soup.get_text())
+
+                links_with_text = []
+                for td_element in soup.find_all('td'):
+                    if td_element.text:
+                        try:
+                            text = td_element.text
+                            split = text.split('SIC')[0]
+                            links_with_text.append(split)
+                        except IndexError:
+                            links_with_text.append(td_element.text)
+
+                for link in links_with_text:
+                    if search_term.lower() in link.lower():
+                        index_of_child.append(len(returnResults))
+                        returnResults.append([{'Company Name': link,
+                                               'Entity Type': 'Company'},
+                                              {uid: {'Resolution': 'Edgar Company',
+                                                     'Notes': ''}}])
+                for code in cikIDs:
+                    returnResults.append([{'CIK': code.split('=')[1],
+                                           'Entity Type': 'Edgar ID'},
+                                          {index_of_child[cikIDs.index(code)]: {'Resolution': 'CIK Edgar ID',
+                                                                                'Notes': ''}}])
+            page.close()
+            browser.close()
+        return returnResults

Modules/EDGAR_&_EFD/EFDByFromDate.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+
+class EFDByFromDate:
+    name = 'Get EFD Reports From Date'
+    description = 'Get EFD reports starting from the date specified by the input entities.'
+    originTypes = {'Date'}
+    resultTypes = {'Politically Exposed Person', 'Website'}
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'},
+                  'To Date': {'description': 'Records will be collected from the Start Date provided by the input '
+                                             'entities. NOTE: The Start Date is assumed to be in ISO format.\n'
+                                             'An End Date is required to complete the Date constraints. '
+                                             'Please input the End Date for the search in the format mm/dd/yyyy',
+                              'type': 'String',
+                              'value': ''},
+                  'Filer Type': {'description': 'Please select the Office you wish to search records for.',
+                                 'type': 'MultiChoice',
+                                 'value': {'Senator',
+                                           'Candidate',
+                                           'Former Senator',
+                                           }},
+                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
+                                  'type': 'MultiChoice',
+                                  'value': {'Annual',
+                                            'Periodic Transactions',
+                                            'Due Date Extension',
+                                            'Blind Trusts',
+                                            'Other Documents',
+                                            }}}
+
+    def resolution(self, entityJsonList, parameters):
+        from datetime import datetime
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
+
+        returnResults = []
+
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter."
+
+        if maxResults <= 0:
+            return []
+
+        try:
+            toDate = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
+        except ValueError:
+            return "Invalid End Date specified."
+
+        url = 'https://efdsearch.senate.gov/search/'
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+
+            pageResolved = False
+            for _ in range(5):
+                try:
+                    page.goto(url, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return "Could not access efdsearch website."
+
+            try:
+                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
+            except TimeoutError:
+                return "The efdsearch website is unresponsive"
+            page.wait_for_timeout(1000)
+
+            for entity in entityJsonList:
+                try:
+                    # Assume ISO format - guessing
+                    date = datetime.fromisoformat(entity['Date'])
+                except ValueError:
+                    continue
+                if toDate < date:
+                    continue
+                date = date.strftime('%m/%d/%Y')
+                toDate = toDate.strftime('%m/%d/%Y')
+                uid = entity['uid']
+                page.wait_for_timeout(1000)
+
+                pageResolved = False
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        pageResolved = True
+                        break
+                    except TimeoutError:
+                        pass
+                if not pageResolved:
+                    continue
+
+                try:
+                    page.fill("input[name=\"submitted_end_date\"]", toDate)
+                    page.fill("input[name=\"submitted_start_date\"]", date)
+                    if 'Senator' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Senator\")")
+                    if 'Candidate' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Candidate\")")
+                    if 'Former Senator' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Former Senator\")")
+
+                    if 'Annual' in parameters['Report Type']:
+                        page.click("text=Annual")
+                    if 'Periodic Transactions' in parameters['Report Type']:
+                        page.click("text=Periodic Transactions")
+                    if 'Due Date Extension' in parameters['Report Type']:
+                        page.click("text=Due Date Extension")
+                    if 'Blind Trusts' in parameters['Report Type']:
+                        page.click("text=Blind Trusts")
+                    if 'Other Documents' in parameters['Report Type']:
+                        page.click("text=Other Documents")
+
+                    page.click("text=Search Reports")
+
+                    entriesInfo = page.locator('#filedReports_info')
+                    entriesInfo.wait_for(state='visible')
+                    currentFirstIndex = 1
+                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+                    resultCount = 0
+
+                    if lastIndex == 0:
+                        continue
+
+                    # Need to click twice to sort by most recent.
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+
+                    while True:
+                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
+                        for record in soup:
+                            if isinstance(record, Tag) and record.get('class'):
+                                recordFields = record.childGenerator()
+                                senateName = next(recordFields).text
+                                senateName += " " + next(recordFields).text
+                                office = next(recordFields).text
+                                report = next(recordFields)
+                                reportType = report.text
+                                reportLink = next(report.children).get('href')
+                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
+                                resultCount += 1
+                                childIndex = len(returnResults)
+                                returnResults.append([{'Full Name': senateName,
+                                                       'Office': office,
+                                                       'Entity Type': 'Politically Exposed Person'},
+                                                      {uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
+                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
+                                                       'Report Type': reportType,
+                                                       'Entity Type': 'Website'},
+                                                      {childIndex: {'Resolution': 'Filed Disclosure Report',
+                                                                    'Notes': '',
+                                                                    'Date Created': dateCreated}}])
+                                if resultCount == maxResults:
+                                    break
+
+                        # Break if we've read enough records, or we ran out of records on this page.
+                        if resultCount == maxResults or currentLastIndex == lastIndex:
+                            break
+
+                        # We've read all the available records, so we click next.
+                        page.click("text=Next")
+                        entriesInfo.wait_for(state='visible')
+                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
+                            page.wait_for_timeout(1000)
+                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
+                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+
+                except TimeoutError:
+                    continue
+                except Exception as e:
+                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
+
+            page.close()
+            browser.close()
+        return returnResults

Modules/EDGAR_&_EFD/EFDByName.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+
+class EFDByName:
+    name = 'Get EFD Reports by Name'
+    description = 'Get EFD reports concerning the people specified by the input entities.'
+    originTypes = {'Person', 'Politically Exposed Person'}
+    resultTypes = {'Website'}
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'},
+                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
+                                  'type': 'MultiChoice',
+                                  'value': {'Annual',
+                                            'Periodic Transactions',
+                                            'Due Date Extension',
+                                            'Blind Trusts',
+                                            'Other Documents',
+                                            }}}
+
+    def resolution(self, entityJsonList, parameters):
+        from datetime import datetime
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
+
+        returnResults = []
+
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter."
+
+        if maxResults <= 0:
+            return []
+
+        url = 'https://efdsearch.senate.gov/search/'
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+
+            pageResolved = False
+            for _ in range(5):
+                try:
+                    page.goto(url, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return "Could not access efdsearch website."
+
+            try:
+                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
+            except TimeoutError:
+                return "The efdsearch website is unresponsive"
+            page.wait_for_timeout(1000)
+
+            for entity in entityJsonList:
+                lastName = entity['Full Name'].split(' ')[-1]
+                firstName = " ".join(entity['Full Name'].split(' ')[:-1])
+
+                uid = entity['uid']
+                page.wait_for_timeout(1000)
+
+                pageResolved = False
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        pageResolved = True
+                        break
+                    except TimeoutError:
+                        pass
+                if not pageResolved:
+                    continue
+
+                try:
+                    personOccupation = entity['Occupation'].lower()
+                    if personOccupation == 'senator':
+                        page.click("label:has-text(\"Senator\")")
+                    elif personOccupation == 'candidate':
+                        page.click("label:has-text(\"Candidate\")")
+                    elif personOccupation == 'former senator':
+                        page.click("label:has-text(\"Former Senator\")")
+
+                    if 'Annual' in parameters['Report Type']:
+                        page.click("text=Annual")
+                    if 'Periodic Transactions' in parameters['Report Type']:
+                        page.click("text=Periodic Transactions")
+                    if 'Due Date Extension' in parameters['Report Type']:
+                        page.click("text=Due Date Extension")
+                    if 'Blind Trusts' in parameters['Report Type']:
+                        page.click("text=Blind Trusts")
+                    if 'Other Documents' in parameters['Report Type']:
+                        page.click("text=Other Documents")
+
+                    page.fill("[placeholder=\"First name (starts with)\"]", firstName)
+                    page.fill("[placeholder=\"Last name (starts with)\"]", lastName)
+
+                    page.click("text=Search Reports")
+
+                    entriesInfo = page.locator('#filedReports_info')
+                    entriesInfo.wait_for(state='visible')
+                    currentFirstIndex = 1
+                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+                    resultCount = 0
+
+                    if lastIndex == 0:
+                        continue
+
+                    # Need to click twice to sort by most recent.
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+
+                    while True:
+                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
+                        for record in soup:
+                            if isinstance(record, Tag) and record.get('class'):
+                                recordFields = record.childGenerator()
+                                senateName = next(recordFields).text
+                                senateName += " " + next(recordFields).text
+                                office = next(recordFields).text
+                                report = next(recordFields)
+                                reportType = report.text
+                                reportLink = next(report.children).get('href')
+                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
+                                resultCount += 1
+                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
+                                                       'Report Type': reportType,
+                                                       'Entity Type': 'Website'},
+                                                      {uid: {'Resolution': 'Filed Disclosure Report',
+                                                             'Notes': '',
+                                                             'Date Created': dateCreated}}])
+                                if resultCount == maxResults:
+                                    break
+
+                        # Break if we've read enough records, or we ran out of records on this page.
+                        if resultCount == maxResults or currentLastIndex == lastIndex:
+                            break
+
+                        # We've read all the available records, so we click next.
+                        page.click("text=Next")
+                        entriesInfo.wait_for(state='visible')
+                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
+                            page.wait_for_timeout(1000)
+                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
+                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+
+                except TimeoutError:
+                    continue
+                except Exception as e:
+                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
+
+            page.close()
+            browser.close()
+        return returnResults

Modules/EDGAR_&_EFD/EFDByToDate.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+
+class EFDByToDate:
+    name = 'Get EFD Reports To Date'
+    description = 'Get EFD reports ending at the date specified by the input entities.'
+    originTypes = {'Date'}
+    resultTypes = {'Politically Exposed Person', 'Website'}
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'},
+                  'To Date': {'description': 'Records will be collected from the End Date provided by the input '
+                                             'entities. NOTE: The End Date is assumed to be in ISO format.\n'
+                                             'A Start Date is required to complete the Date constraints. '
+                                             'Please input the Start Date for the search in the format mm/dd/yyyy',
+                              'type': 'String',
+                              'value': ''},
+                  'Filer Type': {'description': 'Please select the Office you wish to search records for.',
+                                 'type': 'MultiChoice',
+                                 'value': {'Senator',
+                                           'Candidate',
+                                           'Former Senator',
+                                           }},
+                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
+                                  'type': 'MultiChoice',
+                                  'value': {'Annual',
+                                            'Periodic Transactions',
+                                            'Due Date Extension',
+                                            'Blind Trusts',
+                                            'Other Documents',
+                                            }}}
+
+    def resolution(self, entityJsonList, parameters):
+        from datetime import datetime
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
+
+        returnResults = []
+
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter."
+
+        if maxResults <= 0:
+            return []
+
+        try:
+            date = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
+        except ValueError:
+            return "Invalid End Date specified."
+
+        url = 'https://efdsearch.senate.gov/search/'
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+
+            pageResolved = False
+            for _ in range(5):
+                try:
+                    page.goto(url, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+            if not pageResolved:
+                return "Could not access efdsearch website."
+
+            try:
+                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
+            except TimeoutError:
+                return "The efdsearch website is unresponsive"
+            page.wait_for_timeout(1000)
+
+            for entity in entityJsonList:
+                try:
+                    # Assume ISO format - guessing
+                    toDate = datetime.fromisoformat(entity['Date'])
+                except ValueError:
+                    continue
+                if toDate < date:
+                    continue
+                date = date.strftime('%m/%d/%Y')
+                toDate = toDate.strftime('%m/%d/%Y')
+                uid = entity['uid']
+                page.wait_for_timeout(1000)
+
+                pageResolved = False
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        pageResolved = True
+                        break
+                    except TimeoutError:
+                        pass
+                if not pageResolved:
+                    continue
+
+                try:
+                    page.fill("input[name=\"submitted_end_date\"]", toDate)
+                    page.fill("input[name=\"submitted_start_date\"]", date)
+                    if 'Senator' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Senator\")")
+                    if 'Candidate' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Candidate\")")
+                    if 'Former Senator' in parameters['Filer Type']:
+                        page.click("label:has-text(\"Former Senator\")")
+
+                    if 'Annual' in parameters['Report Type']:
+                        page.click("text=Annual")
+                    if 'Periodic Transactions' in parameters['Report Type']:
+                        page.click("text=Periodic Transactions")
+                    if 'Due Date Extension' in parameters['Report Type']:
+                        page.click("text=Due Date Extension")
+                    if 'Blind Trusts' in parameters['Report Type']:
+                        page.click("text=Blind Trusts")
+                    if 'Other Documents' in parameters['Report Type']:
+                        page.click("text=Other Documents")
+
+                    page.click("text=Search Reports")
+
+                    entriesInfo = page.locator('#filedReports_info')
+                    entriesInfo.wait_for(state='visible')
+                    currentFirstIndex = 1
+                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+                    resultCount = 0
+
+                    if lastIndex == 0:
+                        continue
+
+                    # Need to click twice to sort by most recent.
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+                    page.click("text=Date Received/Filed")
+                    page.wait_for_timeout(500)
+
+                    while True:
+                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
+                        for record in soup:
+                            if isinstance(record, Tag) and record.get('class'):
+                                recordFields = record.childGenerator()
+                                senateName = next(recordFields).text
+                                senateName += " " + next(recordFields).text
+                                office = next(recordFields).text
+                                report = next(recordFields)
+                                reportType = report.text
+                                reportLink = next(report.children).get('href')
+                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
+                                resultCount += 1
+                                childIndex = len(returnResults)
+                                returnResults.append([{'Full Name': senateName,
+                                                       'Office': office,
+                                                       'Entity Type': 'Politically Exposed Person'},
+                                                      {uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
+                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
+                                                       'Report Type': reportType,
+                                                       'Entity Type': 'Website'},
+                                                      {childIndex: {'Resolution': 'Filed Disclosure Report',
+                                                                    'Notes': '',
+                                                                    'Date Created': dateCreated}}])
+                                if resultCount == maxResults:
+                                    break
+
+                        # Break if we've read enough records, or we ran out of records on this page.
+                        if resultCount == maxResults or currentLastIndex == lastIndex:
+                            break
+
+                        # We've read all the available records, so we click next.
+                        page.click("text=Next")
+                        entriesInfo.wait_for(state='visible')
+                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
+                            page.wait_for_timeout(1000)
+                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
+                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
+                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
+
+                except TimeoutError:
+                    continue
+                except Exception as e:
+                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
+
+            page.close()
+            browser.close()
+        return returnResults

Modules/EDGAR_&_EFD/Edgar.xml

@@ -0,0 +1,82 @@
+<Edgar>
+    <Edgar_ID>
+        <Attributes>
+            CIK
+        </Attributes>
+    </Edgar_ID>
+    <Form_Field>
+        <Attributes>
+            Field Name,Account Number,Fiscal Year,Fiscal Period,Value,Unit,Taxonomy
+        </Attributes>
+    </Form_Field>
+    <Form13F>
+        <Attributes>
+            Name Of Issuer,Title Of Class,CUSIP,Value,Number Of Shares,Ssh Prnamt Type,Investment Discretion
+        </Attributes>
+    </Form13F>
+    <Form4>
+        <Attributes>
+            Security Title,Deemed Execution Date,Equity Swap Involved,Transaction Timeliness,Transaction Shares,Transaction Price Per Share,Shares Owned Following Transaction,Direct Or Indirect Ownership
+        </Attributes>
+    </Form4>
+    <Form3>
+        <Attributes>
+            Security Title,Shares Owned Following Transaction,Direct Or Indirect Ownership,Nature Of Ownership
+        </Attributes>
+    </Form3>
+    <FormD>
+        <Attributes>
+            Company Name,Industry Group Type,Investment Fund Type,Aggregate Net Asset Value Range,Duration Of Offering,Types Of Securities Offered,Business Combination Transaction,Minimum Investment Accepted,Total Offering Amount,Total Amount Sold,Total Amount Remaining,Has Non Accredited Investors,Total Number Already Invested,Sales Commissions,Finders Fees,Gross Proceeds Used
+        </Attributes>
+    </FormD>
+    <Tickers>
+        <Attributes>
+            Ticker Name
+        </Attributes>
+    </Tickers>
+    <Exchanges>
+        <Attributes>
+            Exchange Name
+        </Attributes>
+    </Exchanges>
+    <SIC>
+        <Attributes>
+            SIC,Description
+        </Attributes>
+    </SIC>
+    <EIN>
+        <Attributes>
+            EIN
+        </Attributes>
+    </EIN>
+    <CUSIP>
+        <Attributes>
+            CUSIP
+        </Attributes>
+    </CUSIP>
+    <LEIID>
+        <Attributes>
+            LEIID
+        </Attributes>
+    </LEIID>
+    <ISINID>
+        <Attributes>
+            ISINID
+        </Attributes>
+    </ISINID>
+    <Shares>
+        <Attributes>
+            Amount
+        </Attributes>
+    </Shares>
+    <FormNMFP2>
+        <Attributes>
+            Field Name,Friday 1,Friday 2,Friday 3,Friday 4,Friday 5
+        </Attributes>
+    </FormNMFP2>
+    <Collateral_Issuer>
+        <Attributes>
+            Name,Coupon or Yield,Principal Amount,Value of Collateral,Ctgry Investments Rprsnts Collateral
+        </Attributes>
+    </Collateral_Issuer>
+</Edgar>

Modules/EDGAR_&_EFD/FramesLookUp.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+class FramesLookUp:
+    # A string that is treated as the name of this resolution.
+    name = "Frames Look Up"
+
+    # A string that describes this resolution.
+    description = "Returns Frame Forms"
+
+    originTypes = {'Form Field'}
+
+    resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Shares'}
+
+    parameters = {
+        'Quarter': {'description': 'Please Ensure that the selected Taxonomy matches the Form Field you typed',
+                    'type': 'SingleChoice',
+                    'value': {'January, February, and March (Q1)', 'April, May, and June (Q2)', 'July, August, and '
+                                                                                                'September (Q3)',
+                              'October, November, and December (Q4)'}},
+        'Year': {'description': 'Please enter the year to match.',
+                 'type': 'String',
+                 'default': '2021'},
+        'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                       'Returns the 5 most recent by default.',
+                        'type': 'String',
+                        'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+
+        year = parameters['Year']
+        quarterChoice = parameters['Quarter']
+        quarter = quarterChoice[quarterChoice.find("(") + 1:quarterChoice.find(")")]
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            unit = entity['Unit']
+            taxonomy = entity['Taxonomy']
+            form_field = entity['Field Name'].split(' ')[1]
+            search_url = f'https://data.sec.gov/api/xbrl/frames/{taxonomy}/{form_field}/{unit}/CY{year}{quarter}I.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+            data = r.json()
+            print(data['data'])
+            if linkNumbers > len(data['data']):
+                linkNumbers = len(data['data'])
+
+            for i in range(linkNumbers):
+                print(data['data'][i])
+                index_of_child = (len(returnResults))
+                returnResults.append([{'Company Name': data['data'][i]['entityName'],
+                                       'Entity Type': 'Company'},
+                                      {uid: {'Resolution': 'Edgar Company',
+                                             'Name': 'Edgar Company',
+                                             'Notes': ''}}])
+
+                returnResults.append([{'CIK': str(data['data'][i]['cik']).zfill(10),
+                                       'Entity Type': 'Edgar ID'},
+                                      {index_of_child: {'Resolution': '',
+                                                        'Name': '',
+                                                        'Notes': ''}}])
+
+                returnResults.append([{'Country Name': data['data'][i]['loc'],
+                                       'Entity Type': 'Country'},
+                                      {index_of_child: {'Resolution': '',
+                                                        'Name': '',
+                                                        'Notes': ''}}])
+                if unit == 'USD':
+                    returnResults.append([{'Amount': str(data['data'][i]['val']),
+                                           'Currency Type': 'USD',
+                                           'Entity Type': 'Currency'},
+                                          {index_of_child: {'Resolution': 'Form Filed Value',
+                                                            'Name': 'Edgar ID',
+                                                            'Notes': ''}}])
+                elif unit == 'shares':
+                    returnResults.append([{'Amount': str(data['data'][i]['val']),
+                                           'Entity Type': 'Shares'},
+                                          {index_of_child: {'Resolution': 'Form Filed Value',
+                                                            'Name': 'Edgar ID',
+                                                            'Notes': ''}}])
+        return returnResults

Modules/EDGAR_&_EFD/Get10KForms.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+class Get10KForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 10-K Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 10-K Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+            print(data)
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '10-K' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '10-K: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Taxonomy': form,
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '10-K Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '10-K' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '10-K: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Taxonomy': form,
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '10-K Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get10QForms.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+class Get10QForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 10-Q Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 10-Q Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '10-Q' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '10-Q: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '10-Q Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '10-Q' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '10-Q: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '10-Q Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get13FForms.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+class Get13FForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 13F Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 13F Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form13F'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns 5 more recent by default',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        import xmltodict
+        import json
+        from bs4 import BeautifulSoup
+        from ast import literal_eval
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        name = ''
+        date = ''
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            archives_set = set()
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
+                         f'={linkNumbers}&type=13F-HR'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            soup = BeautifulSoup(r.text, "lxml")
+
+            for link in soup.find_all('a'):
+                # extract link url from the anchor
+                anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                if '/Archives/edgar/data/' in anchor:
+                    anchor = 'https://www.sec.gov' + anchor
+                    archives_set.add(anchor)
+
+            for archive in archives_set:
+                time.sleep(1)
+                r = requests.get(archive, headers=headers)
+                soup = BeautifulSoup(r.text, "lxml")
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+
+                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                    if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
+                            and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        r = requests.get(anchor, headers=headers)
+                        data = literal_eval(json.dumps(xmltodict.parse(r.text)))
+
+                        date = data['edgarSubmission']['headerData']['filerInfo']['periodOfReport']
+                        name = data['edgarSubmission']['formData']['coverPage']['filingManager']['name']
+
+                    elif '/Archives/edgar/data/' in anchor and 'infotable.xml' in anchor \
+                            and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        r = requests.get(anchor, headers=headers)
+                        data = literal_eval(json.dumps(xmltodict.parse(r.text)))
+
+                        for d in data['informationTable']['infoTable']:
+                            returnResults.append([{'Name Of Issuer': '13F-HR: ' + name + ' ' + d['nameOfIssuer'] + ' '
+                                                                     + date,
+                                                   'Title Of Class': d['titleOfClass'],
+                                                   'CUSIP': d['cusip'],
+                                                   'Value': d['value'],
+                                                   'Number Of Shares': d['shrsOrPrnAmt']['sshPrnamt'],
+                                                   'Ssh Prnamt Type': d['shrsOrPrnAmt']['sshPrnamtType'],
+                                                   'Investment Discretion': d['investmentDiscretion'],
+                                                   'Notes': '',
+
+                                                   'Entity Type': 'Form13F'},
+                                                  {uid: {'Resolution': 'Form13F',
+                                                         'Name': 'Form13F',
+                                                         'Notes': ''}}])
+        return returnResults

Modules/EDGAR_&_EFD/Get20FForms.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+class Get20FForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 20-F Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 20-F Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            print(cik)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+            data = r.json()
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '20-F' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '20-F: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Taxonomy': form,
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '20-F Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '20-F' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '20-F: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Taxonomy': form,
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '20-F Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get3Forms.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+class Get3Forms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 3 Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 3 Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Person, Form3'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        import xmltodict
+        import json
+        from bs4 import BeautifulSoup
+        from ast import literal_eval
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            archives_set = set()
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
+                         f'={linkNumbers}&type=3'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            soup = BeautifulSoup(r.text, "lxml")
+
+            for link in soup.find_all('a'):
+                # extract link url from the anchor
+                anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                if '/Archives/edgar/data/' in anchor:
+                    anchor = 'https://www.sec.gov' + anchor
+                    archives_set.add(anchor)
+
+            for archive in archives_set:
+                time.sleep(1)
+                r = requests.get(archive, headers=headers)
+                soup = BeautifulSoup(r.text, "lxml")
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                    if '/Archives/edgar/data/' in anchor and 'ownership.xml' in anchor and 'xslF345X02' not in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        r = requests.get(anchor, headers=headers)
+                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
+                        data = literal_eval(data)
+                        print(data)
+
+                        name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
+                        remarks = \
+                            data['ownershipDocument']['reportingOwner']['reportingOwnerRelationship']['officerTitle']
+                        value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
+                        index_of_child = len(returnResults)
+                        returnResults.append([{'Full Name': name,
+                                               'Notes': remarks,
+                                               'Entity Type': 'Person'},
+                                              {uid: {'Resolution': 'Reporting Owner',
+                                                     'Notes': ''}}])
+
+                        if value['ownershipNature']['directOrIndirectOwnership']['value'] == 'I':
+                            nature = 'Indirect'
+                        else:
+                            nature = 'Direct'
+                        returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' ' +
+                                                                 data['ownershipDocument']['ownerSignature'][
+                                                                     'signatureDate'],
+                                               'Shares Owned Following Transaction':
+                                                   value['postTransactionAmounts']['sharesOwnedFollowingTransaction'][
+                                                       'value'],
+                                               'Direct Or Indirect Ownership': nature,
+                                               'Nature Of Ownership':
+                                                   value['ownershipNature']['natureOfOwnership']['value'],
+                                               'Notes': '',
+                                               'Entity Type': 'Form3'},
+                                              {index_of_child: {'Resolution': 'Form 3',
+                                                                'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get40FForms.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+class Get40FForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 40-F Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 40-F Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            print(r.content)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '40-F' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '40-F: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Taxonomy': form,
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '40-F Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '40-F' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '40-F: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Taxonomy': form,
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '40-F Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get4Forms.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+class Get4Forms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 4 Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes D Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Person, Form4'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        import xmltodict
+        import json
+        from bs4 import BeautifulSoup
+        from ast import literal_eval
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            archives_set = set()
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
+                         f'={linkNumbers}&type=4'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            soup = BeautifulSoup(r.text, "lxml")
+
+            for link in soup.find_all('a'):
+                # extract link url from the anchor
+                anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                if '/Archives/edgar/data/' in anchor:
+                    anchor = 'https://www.sec.gov' + anchor
+                    archives_set.add(anchor)
+
+            for archive in archives_set:
+                time.sleep(1)
+                r = requests.get(archive, headers=headers)
+                soup = BeautifulSoup(r.text, "lxml")
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                    if '/Archives/edgar/data/' in anchor and '.xml' in anchor and 'xslF345X03' not in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        r = requests.get(anchor, headers=headers)
+                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
+                        data = literal_eval(data)
+                        print(data)
+
+                        index_of_child = len(returnResults)
+                        try:
+                            remarks = data['ownershipDocument']['remarks']
+                        except KeyError:
+                            remarks = ''
+                        name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
+                        returnResults.append([{'Full Name': name,
+                                               'Notes': remarks,
+                                               'Entity Type': 'Person'},
+                                              {uid: {'Resolution': 'Reporting Owner',
+                                                     'Notes': ''}}])
+
+                        if type(data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']) == dict:
+                            value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
+                            try:
+                                footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
+                            except KeyError:
+                                footnote = value['transactionAmounts']['transactionShares']['value']
+                            try:
+                                footnotePerShare = \
+                                    value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
+                            except KeyError:
+                                footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
+                            returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
+                                                                     + value['transactionCoding'][
+                                                                         'transactionCode'] + ' ' +
+                                                                     data['ownershipDocument']['ownerSignature'][
+                                                                         'signatureDate'],
+                                                   'Deemed Execution Date': str(value['deemedExecutionDate']),
+                                                   'Equity Swap Involved': value['transactionCoding'][
+                                                       'equitySwapInvolved'],
+                                                   'Transaction Timeliness': str(value['transactionTimeliness']),
+                                                   'Transaction Shares': footnote,
+                                                   'Transaction Price Per Share': footnotePerShare,
+                                                   'Shares Owned Following Transaction':
+                                                       value['postTransactionAmounts'][
+                                                           'sharesOwnedFollowingTransaction'],
+                                                   'Notes': (': '.join(
+                                                       map(str, data['ownershipDocument']['footnotes']['footnote']))),
+                                                   'Entity Type': 'Form4'},
+                                                  {index_of_child: {'Resolution': 'Form 4',
+                                                                    'Notes': ''}}])
+                        else:
+
+                            for value in data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']:
+                                try:
+                                    footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
+                                except KeyError:
+                                    footnote = value['transactionAmounts']['transactionShares']['value']
+                                try:
+                                    footnotePerShare = \
+                                        value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
+                                except KeyError:
+                                    footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
+                                returnResults.append(
+                                    [{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
+                                                        + value['transactionCoding']['transactionCode'] + ' ' +
+                                                        data['ownershipDocument']['ownerSignature']['signatureDate'],
+                                      'Deemed Execution Date': str(value['deemedExecutionDate']),
+                                      'Equity Swap Involved': value['transactionCoding']['equitySwapInvolved'],
+                                      'Transaction Timeliness': str(value['transactionTimeliness']),
+                                      'Transaction Shares': footnote,
+                                      'Transaction Price Per Share': footnotePerShare,
+                                      'Shares Owned Following Transaction':
+                                          value['postTransactionAmounts']['sharesOwnedFollowingTransaction']['value'],
+                                      'Notes': (
+                                          ': '.join(map(str, data['ownershipDocument']['footnotes']['footnote']))),
+                                      'Entity Type': 'Form4'},
+                                     {index_of_child: {'Resolution': 'Form 4',
+                                                       'Notes': ''}}])
+        return returnResults

Modules/EDGAR_&_EFD/Get6KForms.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+class Get6KForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 6-K Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 6-K Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            print(cik)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            print(r.content)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+            print(data)
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '6-K' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '6-K: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Taxonomy': form,
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '6-K Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '6-K' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '6-K: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Taxonomy': form,
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '6-K Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/Get8KForms.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+class Get8KForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent 8-K Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes 8-K Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Form Field'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            print(cik)
+            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            print(r.content)
+            if r.status_code != 200:
+                return []
+
+            data = r.json()
+            print(data)
+
+            forms = list(data['facts'].keys())
+
+            for form in forms:
+                keys = list(data['facts'][form].keys())
+                for i in keys:
+                    if 'Deprecated' not in data['facts'][form][i]['label']:
+                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
+                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
+                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
+                            for j in range(linkNumbers):
+                                if '8-K' in data['facts'][form][i]['units']['USD'][j]['form']:
+                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
+                                    returnResults.append([{'Field Name': '8-K: ' + i + ' ' + value['filed'],
+                                                           'Account Number': value['accn'],
+                                                           'Fiscal Year': value['fy'],
+                                                           'Fiscal Period': value['fp'],
+                                                           'Value': value['val'],
+                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                           'Taxonomy': form,
+                                                           'Notes': data['facts'][form][i]['label'],
+
+                                                           'Entity Type': 'Form Field'},
+                                                          {uid: {'Resolution': '8-K Field',
+                                                                 'Name': 'CIK Edgar ID',
+                                                                 'Notes': ''}}])
+
+                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
+                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
+                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
+                        for j in range(linkNumbers):
+                            if '8-K' in data['facts'][form][i]['units']['shares'][j]['form']:
+                                value = data['facts'][form][i]['units']['shares'][::-1][j]
+                                returnResults.append([{'Field Name': '8-K: ' + i + ' ' + value['filed'],
+                                                       'Account Number': value['accn'],
+                                                       'Fiscal Year': value['fy'],
+                                                       'Fiscal Period': value['fp'],
+                                                       'Value': value['val'],
+                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
+                                                       'Taxonomy': form,
+                                                       'Notes': data['facts'][form][i]['label'],
+
+                                                       'Entity Type': 'Form Field'},
+                                                      {uid: {'Resolution': '8-K Field',
+                                                             'Name': 'CIK Edgar ID',
+                                                             'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/GetDForms.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+class GetDForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent D Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes D Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'FormD, Person, Address, Phrase'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+        import xmltodict
+        import json
+        from bs4 import BeautifulSoup
+        from ast import literal_eval
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            archives_set = set()
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
+                         f'={linkNumbers}&type=D'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            soup = BeautifulSoup(r.text, "lxml")
+
+            for link in soup.find_all('a'):
+                # extract link url from the anchor
+                anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                if '/Archives/edgar/data/' in anchor:
+                    anchor = 'https://www.sec.gov' + anchor
+                    archives_set.add(anchor)
+
+            for archive in archives_set:
+                time.sleep(1)
+                r = requests.get(archive, headers=headers)
+                soup = BeautifulSoup(r.text, "lxml")
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                    if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
+                            and 'xslFormDX01' not in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        r = requests.get(anchor, headers=headers)
+                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
+                        data = literal_eval(data)
+                        print(data)
+
+                        value = data['edgarSubmission']['offeringData']
+                        index_of_child = len(returnResults)
+                        returnResults.append([{'Company Name': 'D: ' + data['edgarSubmission']['primaryIssuer']
+                        ['entityName'] + ' ' + value['signatureBlock']['signature']['signatureDate'],
+                                               'Industry Group Type': value['industryGroup']['industryGroupType'],
+                                               'Investment Fund Type': value['industryGroup']['investmentFundInfo']
+                                               ['investmentFundType'],
+                                               'Aggregate Net Asset Value Range': value['issuerSize']
+                                               ['aggregateNetAssetValueRange'],
+                                               'Duration Of Offering': 'More Than one Year: ' +
+                                                                       value['durationOfOffering']['moreThanOneYear'],
+                                               'Types Of Securities Offered': 'Pooled Investment Fund Type: ' +
+                                                                              value['typesOfSecuritiesOffered'][
+                                                                                  'isPooledInvestmentFundType'],
+                                               'Business Combination Transaction': 'Business Combination Transaction: '
+                                                                                   + value[
+                                                                                       'businessCombinationTransaction'][
+                                                                                       'isBusinessCombinationTransaction'],
+                                               'Minimum Investment Accepted': value['minimumInvestmentAccepted'],
+                                               'Total Offering Amount': value['offeringSalesAmounts']
+                                               ['totalOfferingAmount'],
+                                               'Total Amount Sold': value['offeringSalesAmounts']['totalAmountSold'],
+                                               'Total Amount Remaining': value['offeringSalesAmounts'][
+                                                   'totalRemaining'],
+                                               'Has Non Accredited Investors': 'Non Accredited Investors'
+                                                                               + value['investors'][
+                                                                                   'hasNonAccreditedInvestors'],
+                                               'Total Number Already Invested': value['investors']
+                                               ['totalNumberAlreadyInvested'],
+                                               'Sales Commissions': value['salesCommissionsFindersFees']
+                                               ['salesCommissions']['dollarAmount'],
+                                               'Finders Fees': value['salesCommissionsFindersFees']['findersFees']
+                                               ['dollarAmount'],
+                                               'Gross Proceeds Used': value['useOfProceeds']['grossProceedsUsed']
+                                               ['dollarAmount'],
+                                               'Notes': '',
+
+                                               'Entity Type': 'FormD'},
+                                              {uid: {'Resolution': 'D Form',
+                                                     'Name': 'D Form',
+                                                     'Notes': ''}}])
+
+                        people = data['edgarSubmission']['relatedPersonsList']['relatedPersonInfo']
+                        for person in people:
+                            child_of_child = len(returnResults)
+                            returnResults.append([{'Full Name': person['relatedPersonName']['firstName'] + ' ' +
+                                                                person['relatedPersonName']['lastName'],
+                                                   'Entity Type': 'Person'},
+                                                  {index_of_child: {'Resolution': 'Officer',
+                                                                    'Notes': ''}}])
+
+                            returnResults.append(
+                                [{'Street Address': person['relatedPersonAddress']['street1'],
+                                  'Locality': person['relatedPersonAddress']['city'],
+                                  'Postal Code': person['relatedPersonAddress']['zipCode'],
+                                  'Country': person['relatedPersonAddress']['stateOrCountryDescription'],
+                                  'Entity Type': 'Address'},
+                                 {child_of_child: {'Resolution': 'Location', 'Notes': ''}}])
+
+                            returnResults.append(
+                                [{'Phrase': person['relatedPersonRelationshipList']['relationship'],
+                                  'Entity Type': 'Phrase'},
+                                 {child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
+
+                            if person['relationshipClarification'] is not None:
+                                returnResults.append(
+                                    [{'Phrase': person['relationshipClarification'],
+                                      'Entity Type': 'Phrase'},
+                                     {child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
+
+        return returnResults

Modules/EDGAR_&_EFD/GetN8FForms.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+class GetN8FForms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent N-8F Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes N-8F Forms Websites"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Website'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import time
+
+        from bs4 import BeautifulSoup
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        }
+
+        try:
+            linkNumbers = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer provided in 'Max Results' parameter"
+        if linkNumbers <= 0:
+            return []
+        returnResults = []
+        for entity in entityJsonList:
+            archives_set = set()
+            uid = entity['uid']
+            cik = entity['CIK']
+            if cik.lower().startswith('cik'):
+                cik = cik.split('cik')[1]
+            if len(cik) != 10:
+                cik = cik.zfill(10)
+            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
+                         f'={linkNumbers}&type=N-8F'
+            time.sleep(1)
+            r = requests.get(search_url, headers=headers)
+            if r.status_code != 200:
+                return []
+
+            soup = BeautifulSoup(r.text, "lxml")
+
+            for link in soup.find_all('a'):
+                # extract link url from the anchor
+                anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                if '/Archives/edgar/data/' in anchor:
+                    anchor = 'https://www.sec.gov' + anchor
+                    archives_set.add(anchor)
+
+            for archive in archives_set:
+                time.sleep(1)
+                r = requests.get(archive, headers=headers)
+                soup = BeautifulSoup(r.text, "lxml")
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
+                    if '/Archives/edgar/data/' in anchor and '.htm' in anchor:
+                        time.sleep(1)
+                        anchor = 'https://www.sec.gov' + anchor
+                        returnResults.append([{'URL': anchor,
+                                               'Entity Type': 'Website'},
+                                              {uid: {'Resolution': 'N-8F Form',
+                                                     'Name': 'N-8F Form',
+                                                     'Notes': ''}}])
+        return returnResults

Modules/EDGAR_&_EFD/GetNMFP2Forms.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+
+class GetNMFP2Forms:
+    # A string that is treated as the name of this resolution.
+    name = "Get Recent N-MFP2 Forms"
+
+    # A string that describes this resolution.
+    description = "Returns Nodes N-MFP2 Forms"
+
+    originTypes = {'Edgar ID'}
+
+    resultTypes = {'Collateral Issuer, Company, Phrase, FormNMFP2, CUSIP, LEIID, ISINID'}
+
+    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
+                                                 'Returns the 5 most recent by default.',
+                                  'type': 'String',
+                                  'default': '5'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+        import xmltodict
+        import json
+        from playwright.sync_api import sync_playwright, TimeoutError
+        from bs4 import BeautifulSoup
+        from ast import literal_eval
+
+        headers = {
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
+        }
+
+        try:
+            maxResults = int(parameters['Max Results'])
+        except ValueError:
+            return "Invalid integer value provided for 'Max Results' parameter."
+        if maxResults <= 0:
+            return []
+
+        returnResults = []
+        liquidAssets = ['totalValueDailyLiquidAssets', 'totalValueWeeklyLiquidAssets', 'percentageDailyLiquidAssets',
+                        'percentageWeeklyLiquidAssets', 'netAssetValue']
+        seriesLevelInfoKeys = ['feederFundFlag', 'masterFundFlag', 'seriesFundInsuCmpnySepAccntFlag',
+                               'fundExemptRetailFlag', 'averagePortfolioMaturity',
+                               'averageLifeMaturity', 'cash',
+                               'totalValuePortfolioSecurities', 'amortizedCostPortfolioSecurities',
+                               'totalValueOtherAssets', 'totalValueLiabilities', 'netAssetOfSeries',
+                               'numberOfSharesOutstanding', 'stablePricePerShare', 'sevenDayGrossYield']
+        classLevelInfoKeys = ['minInitialInvestment', 'netAssetsOfClass', 'numberOfSharesOutstanding',
+                              'sevenDayNetYield', 'personPayForFundFlag']
+        securitiesInfoKeys = ['titleOfIssuer', 'investmentCategory', 'securityEligibilityFlag',
+                              'investmentMaturityDateWAM', 'investmentMaturityDateWAL',
+                              'finalLegalInvestmentMaturityDate', 'securityDemandFeatureFlag', 'securityGuaranteeFlag',
+                              'securityEnhancementsFlag', 'yieldOfTheSecurityAsOfReportingDate',
+                              'includingValueOfAnySponsorSupport', 'excludingValueOfAnySponsorSupport',
+                              'percentageOfMoneyMarketFundNetAssets', 'securityCategorizedAtLevel3Flag',
+                              'dailyLiquidAssetSecurityFlag', 'weeklyLiquidAssetSecurityFlag', 'illiquidSecurityFlag']
+
+        with sync_playwright() as p:
+            browser = p.firefox.launch()
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+            )
+            page = context.new_page()
+            for entity in entityJsonList:
+                archives_set = set()
+                uid = entity['uid']
+                cik = entity['CIK']
+                if cik.lower().startswith('cik'):
+                    cik = cik.split('cik')[1]
+                if len(cik) != 10:
+                    cik = cik.zfill(10)
+                search_url = f'https://www.sec.gov/edgar/search/#/category=custom&entityName={cik}&forms=N-MFP2'
+                page.wait_for_timeout(1000)
+                pageResolved = False
+                for _ in range(3):
+                    try:
+                        page.goto(search_url, wait_until="networkidle", timeout=10000)
+                        pageResolved = True
+                        break
+                    except TimeoutError:
+                        pass
+                if not pageResolved:
+                    continue
+                page.wait_for_timeout(1000)
+
+                soup = BeautifulSoup(page.content(), "lxml")
+
+                for link in soup.find_all('a'):
+                    # extract link url from the anchor
+                    anchor = link.attrs['data-adsh'] if 'data-adsh' in link.attrs else ''
+                    if anchor != '':
+                        anchor = anchor.replace('-', '')
+                        anchor = f'https://www.sec.gov/Archives/edgar/data/{cik}/{anchor}/primary_doc.xml'
+                        archives_set.add(anchor)
+
+                for link in range(maxResults):
+                    r = requests.get(list(archives_set)[link], headers=headers)
+                    data = literal_eval(json.dumps(xmltodict.parse(r.text)).replace('null', 'None'))
+                    fieldPath = data['edgarSubmission']['formData']['seriesLevelInfo']
+                    seriesId = data['edgarSubmission']['formData']['generalInfo'][
+                        'seriesId']
+                    date = data['edgarSubmission']['formData']['generalInfo'][
+                        'reportDate']
+
+                    returnResults.append(
+                        [{'Company Name': fieldPath['adviser']['adviserName'],
+                          'Entity Type': 'Company'},
+                         {uid: {'Resolution': 'Adviser', 'Notes': ''}}])
+                    returnResults.append(
+                        [{'Company Name': fieldPath['indpPubAccountant']['name'],
+                          'Entity Type': 'Company'},
+                         {uid: {'Resolution': 'Independent Pub Accountant', 'Notes': ''}}])
+                    returnResults.append(
+                        [{'Company Name': fieldPath['administrator']['administratorName'],
+                          'Entity Type': 'Company'},
+                         {uid: {'Resolution': 'Administrator', 'Notes': ''}}])
+                    returnResults.append(
+                        [{'Company Name': fieldPath['transferAgent']['name'],
+                          'Entity Type': 'Company'},
+                         {uid: {'Resolution': 'Transfer Agent', 'Notes': ''}}])
+
+                    for value in seriesLevelInfoKeys:
+                        returnResults.append(
+                            [{'Phrase': f'N-MFP2:({value}) '
+                                        + f'ID: {seriesId} Date: {date}',
+                              'Notes': fieldPath[value],
+                              'Entity Type': 'Phrase'},
+                             {uid: {'Resolution': value, 'Notes': ''}}])
+
+                    for value in fieldPath['moneyMarketFundCategory']:
+                        returnResults.append(
+                            [{'Phrase': value,
+                              'Entity Type': 'Phrase'},
+                             {uid: {'Resolution': value, 'Notes': ''}}])
+
+                    for field in liquidAssets:
+                        if 'Daily' in field:
+                            returnResults.append([{'Field Name': f'N-MFP2:({field})' + ' '
+                                                                 + f'ID: {seriesId} Date: {date}',
+                                                   'Friday 1': fieldPath[field][
+                                                       'ns3:fridayDay1'],
+                                                   'Friday 2': fieldPath[field][
+                                                       'ns3:fridayDay2'],
+                                                   'Friday 3': fieldPath[field][
+                                                       'ns3:fridayDay3'],
+                                                   'Friday 4': fieldPath[field][
+                                                       'ns3:fridayDay4'],
+                                                   'Friday 5': 'NO Value in Daily Measure',
+                                                   'Entity Type': 'FormNMFP2'},
+                                                  {uid: {'Resolution': field,
+                                                         'Name': 'FormNMFP2',
+                                                         'Notes': ''}}])
+                        else:
+                            returnResults.append([{'Field Name': f'N-MFP2:({field})' + ' '
+                                                                 + f'ID: {seriesId} Date: {date}',
+                                                   'Friday 1': fieldPath[field][
+                                                       'ns3:fridayWeek1'],
+                                                   'Friday 2': fieldPath[field][
+                                                       'ns3:fridayWeek2'],
+                                                   'Friday 3': fieldPath[field][
+                                                       'ns3:fridayWeek3'],
+                                                   'Friday 4': fieldPath[field][
+                                                       'ns3:fridayWeek4'],
+                                                   'Friday 5': fieldPath[field][
+                                                       'ns3:fridayWeek5'],
+                                                   'Entity Type': 'FormNMFP2'},
+                                                  {uid: {'Resolution': field,
+                                                         'Name': 'FormNMFP2',
+                                                         'Notes': ''}}])
+
+                    classLevelInfo = data['edgarSubmission']['formData']['classLevelInfo']
+                    for classInfo in classLevelInfo:
+                        index_of_child = len(returnResults)
+                        returnResults.append(
+                            [{'Phrase': classInfo['classesId'],
+                              'Entity Type': 'Phrase'},
+                             {uid: {'Resolution': 'Classes Id', 'Notes': ''}}])
+                        returnResults.append([{'Field Name': f'N-MFP2:(Net Asset Per Share)' + ' '
+                                                             + f'ID: {seriesId} Date: {date}',
+                                               'Friday 1': classInfo['netAssetPerShare'][
+                                                   'ns3:fridayWeek1'],
+                                               'Friday 2': classInfo['netAssetPerShare'][
+                                                   'ns3:fridayWeek2'],
+                                               'Friday 3': classInfo['netAssetPerShare'][
+                                                   'ns3:fridayWeek3'],
+                                               'Friday 4': classInfo['netAssetPerShare'][
+                                                   'ns3:fridayWeek4'],
+                                               'Friday 5': classInfo['netAssetPerShare'][
+                                                   'ns3:fridayWeek5'],
+                                               'Entity Type': 'FormNMFP2'},
+                                              {index_of_child: {'Resolution': 'Net Asset Per Share',
+                                                                'Name': 'FormNMFP2',
+                                                                'Notes': ''}}])
+                        for weekCount in range(1, 6):
+                            returnResults.append(
+                                [{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossSubscriptions'],
+                                  'Entity Type': 'Phrase'},
+                                 {index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Subscriptions',
+                                                   'Notes': ''}}])
+                            returnResults.append(
+                                [{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossRedemptions'],
+                                  'Entity Type': 'Phrase'},
+                                 {index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Redemptions',
+                                                   'Notes': ''}}])
+
+                        for value in classLevelInfoKeys:
+                            returnResults.append(
+                                [{'Phrase': classInfo[value],
+                                  'Entity Type': 'Phrase'},
+                                 {index_of_child: {'Resolution': value,
+                                                   'Notes': ''}}])
+
+                    scheduleOfPortfolioSecurities = data['edgarSubmission']['formData']['scheduleOfPortfolioSecuritiesInfo']
+                    instance = 0
+                    for securitiesInfo in scheduleOfPortfolioSecurities:
+                        index_of_child = len(returnResults)
+                        returnResults.append(
+                            [{'Company Name': securitiesInfo.get('nameOfIssuer') + ' ' + str(instance),
+                              'Entity Type': 'Company'},
+                             {uid: {'Resolution': 'Issuer', 'Notes': ''}}])
+                        instance += 1
+                        returnResults.append(
+                            [{'CUSIP': securitiesInfo.get('CUSIPMember'),
+                              'Entity Type': 'CUSIP'},
+                             {index_of_child: {'Resolution': 'CUSIP', 'Notes': ''}}])
+                        returnResults.append(
+                            [{'LEIID': securitiesInfo.get('LEIID'),
+                              'Entity Type': 'LEIID'},
+                             {index_of_child: {'Resolution': 'LEIID', 'Notes': ''}}])
+                        returnResults.append(
+                            [{'ISINID': securitiesInfo.get('ISINId'),
+                              'Entity Type': 'ISINID'},
+                             {index_of_child: {'Resolution': 'ISINID', 'Notes': ''}}])
+
+                        for value in securitiesInfoKeys:
+                            returnResults.append(
+                                [{'Phrase': securitiesInfo[value],
+                                  'Entity Type': 'Phrase'},
+                                 {index_of_child: {'Resolution': value,
+                                                   'Notes': ''}}])
+
+                        for value in securitiesInfo['NRSRO']:
+                            child_of_child = len(returnResults)
+                            returnResults.append(
+                                [{'Company Name': value.get('nameOfNRSRO'),
+                                  'Entity Type': 'Company'},
+                                 {index_of_child: {'Resolution': 'NRSRO',
+                                                   'Notes': ''}}])
+                            returnResults.append(
+                                [{'Phrase': value.get('rating'),
+                                  'Entity Type': 'Phrase'},
+                                 {child_of_child: {'Resolution': 'Rating',
+                                                   'Notes': ''}}])
+
+                        try:
+                            collateralIssuer = securitiesInfo['collateralIssuers']
+                            for issuer in collateralIssuer:
+                                returnResults.append([{'Name': issuer['nameOfCollateralIssuer'],
+                                                       'Coupon or Yield': issuer['couponOrYield'],
+                                                       'Principal Amount': issuer['principalAmountToTheNearestCent'],
+                                                       'Value of Collateral': issuer['valueOfCollateralToTheNearestCent'],
+                                                       'Ctgry Investments Rprsnts Collateral':
+                                                           issuer['ctgryInvestmentsRprsntsCollateral'],
+                                                       'Entity Type': 'Collateral Issuer'},
+                                                      {index_of_child: {'Resolution': 'Net Asset Per Share',
+                                                                        'Name': 'Collateral Issuer',
+                                                                        'Notes': ''}}])
+
+                        except KeyError:
+                            continue
+            page.close()
+            browser.close()
+        return returnResults

Modules/EDGAR_&_EFD/requirements.txt

@@ -0,0 +1,3 @@
+requests
+beautifulsoup4
+xmltodict

Modules/ExampleModule/ExampleModuleEntities.xml

@@ -0,0 +1,10 @@
+<ExampleEntities>
+  <Example>
+    <Attributes>
+      ExampleLabel
+    </Attributes>
+    <Icon>
+      Default.svg
+    </Icon>
+  </Example>
+</ExampleEntities>