Bump python to 3.13, and app version

Convert | to _

Another bug fix & bump version

.gitignore

@@ -0,0 +1,6 @@
+/magic/COPYING.file
+/magic/COPYING.libgnurx
+/magic/file.exe
+/magic/libgnurx-0.dll
+/magic/libmagic-1.dll
+/magic/magic.mgc

Core/Entities/Devices.xml

@@ -1,7 +1,8 @@
 <Devices>
     <Mobile_Phone>
         <Attributes>
-            Device UID,Brand
+            <Attribute default="a:b:c:d:e:f" check="String" primary="True">Device UID</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
         </Attributes>
         <Icon>
             Mobile_Device.svg
@@ -9,7 +10,8 @@
     </Mobile_Phone>
     <Computer>
         <Attributes>
-            Hostname,Brand
+            <Attribute default="User Workstation" check="String" primary="True">Hostname</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
         </Attributes>
         <Icon>
             Computer.svg
@@ -17,7 +19,8 @@
     </Computer>
     <Camera>
         <Attributes>
-            Device UID,Brand
+            <Attribute default="a:b:c:d:e:f" check="String" primary="True">Device UID</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Brand</Attribute>
         </Attributes>
         <Icon>
             Camera.svg

Core/Entities/Financials.xml

@@ -1,7 +1,9 @@
 <Financials>
     <Bank_Account>
         <Attributes>
-            Account Number,Bank Name,Branch Number
+            <Attribute default="Account Number" check="String" primary="True">Account Number</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Bank Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Branch Number</Attribute>
         </Attributes>
         <Icon>
             Bank_Account.svg
@@ -9,10 +11,36 @@
     </Bank_Account>
     <Currency>
         <Attributes>
-            Amount,Currency Type
+            <Attribute default="Money Amount" check="String" primary="True">Amount</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Currency Type</Attribute>
         </Attributes>
         <Icon>
             Currency.svg
         </Icon>
     </Currency>
+    <Ticker>
+        <Attributes>
+            <Attribute default="Ticker ID" check="String" primary="True">Ticker ID</Attribute>
+        </Attributes>
+        <Icon>
+            Share.svg
+        </Icon>
+    </Ticker>
+    <Portfolio>
+        <Attributes>
+            <Attribute default="Portfolio Name" check="String" primary="True">Label</Attribute>
+        </Attributes>
+        <Icon>
+            Portfolio.svg
+        </Icon>
+    </Portfolio>
+    <Crypto_Wallet>
+        <Attributes>
+          <Attribute default="Wallet Address" check="String" primary="True">Wallet Address</Attribute>
+          <Attribute default="Bitcoin" check="String" primary="False">Currency Name</Attribute>
+        </Attributes>
+        <Icon>
+          CryptoWallet.svg
+        </Icon>
+    </Crypto_Wallet>
 </Financials>

Core/Entities/Groups.xml

@@ -1,7 +1,8 @@
 <Collectives>
     <Company>
         <Attributes>
-            Company Name,Registration Number
+            <Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Registration Number</Attribute>
         </Attributes>
         <Icon>
             Company.svg
@@ -9,7 +10,8 @@
     </Company>
     <Organization>
         <Attributes>
-            Organization Name,Registration Number
+            <Attribute default="Organization Name" check="String" primary="True">Organization Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Registration Number</Attribute>
         </Attributes>
         <Icon>
             Organization.svg
@@ -17,7 +19,8 @@
     </Organization>
     <Cult>
         <Attributes>
-            Cult Name,Ideology
+            <Attribute default="Default Cult" check="String" primary="True">Cult Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Ideology</Attribute>
         </Attributes>
         <Icon>
             Cult.svg
@@ -25,7 +28,8 @@
     </Cult>
     <Terrorist_Group>
         <Attributes>
-            Group Name,Ideology
+            <Attribute default="Terror Cell Name" check="String" primary="True">Group Name</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Ideology</Attribute>
         </Attributes>
         <Icon>
             Terrorists.svg
@@ -33,7 +37,7 @@
     </Terrorist_Group>
     <Family>
         <Attributes>
-            Family Surname
+            <Attribute default="Mr. and Mrs. Smith" check="String" primary="True">Family Surname</Attribute>
         </Attributes>
         <Icon>
             Family.svg

Core/Entities/Identifiers.xml

@@ -1,7 +1,7 @@
 <Identifiers>
     <Identification_Number>
         <Attributes>
-            ID Number
+            <Attribute default="ID Number" check="String" primary="True">ID Number</Attribute>
         </Attributes>
         <Icon>
             ID_Number.svg
@@ -9,7 +9,7 @@
     </Identification_Number>
     <Passport_Number>
         <Attributes>
-            Passport Number
+            <Attribute default="Passport Number" check="String" primary="True">Passport Number</Attribute>
         </Attributes>
         <Icon>
             Passport.svg
@@ -17,7 +17,7 @@
     </Passport_Number>
     <Vehicle_Registration>
         <Attributes>
-            Registration Number
+            <Attribute default="Vehicle Registration Number" check="String" primary="True">Registration Number</Attribute>
         </Attributes>
         <Icon>
             VehicleRegistration.svg
@@ -25,7 +25,8 @@
     </Vehicle_Registration>
     <Flight_Number>
         <Attributes>
-            Flight Number,Airline
+            <Attribute default="Flight Number" check="String" primary="True">Flight Number</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Airline</Attribute>
         </Attributes>
         <Icon>
             Flight_Number.svg

Core/Entities/Individuals.xml

@@ -1,7 +1,11 @@
 <Individuals>
   <Person>
     <Attributes>
-      Full Name,Gender,Occupation,Date of Birth
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
     </Attributes>
     <Icon>
       Person.svg
@@ -9,15 +13,19 @@
   </Person>
   <Politically_Exposed_Person>
     <Attributes>
-      Full Name,Gender,Occupation,Date of Birth
+      <Attribute default="Firstname Lastname" check="String" primary="True">Full Name</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Gender</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Occupation</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Date of Birth</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Nationality</Attribute>
     </Attributes>
     <Icon>
-      Person.svg
+      PEP.svg
     </Icon>
   </Politically_Exposed_Person>
   <Phrase>
     <Attributes>
-      Phrase
+      <Attribute default="Phrase" check="String" primary="True">Phrase</Attribute>
     </Attributes>
     <Icon>
       Phrase.svg
@@ -25,7 +33,7 @@
   </Phrase>
   <Sentiment>
     <Attributes>
-      Sentiment
+      <Attribute default="Positive" check="String" primary="True">Sentiment</Attribute>
     </Attributes>
     <Icon>
       Sentiment.svg
@@ -33,7 +41,7 @@
   </Sentiment>
   <Email_Address>
     <Attributes>
-      Email Address
+      <Attribute default="example@example.com" check="Email" primary="True">Email Address</Attribute>
     </Attributes>
     <Icon>
       Email.svg
@@ -41,7 +49,7 @@
   </Email_Address>
   <Phone_Number>
     <Attributes>
-      Phone Number
+      <Attribute default="911" check="Phonenumber" primary="True">Phone Number</Attribute>
     </Attributes>
     <Icon>
       Phone_Number.svg
@@ -49,7 +57,7 @@
   </Phone_Number>
   <Date>
     <Attributes>
-      Date
+      <Attribute default="1997/1/1" check="String" primary="True">Date</Attribute>
     </Attributes>
     <Icon>
       Date.svg

Core/Entities/Infrastructure.xml

@@ -1,7 +1,7 @@
 <Infrastructure>
   <Operating_System>
     <Attributes>
-      OS Name
+      <Attribute default="Operating System" check="String" primary="True">OS Name</Attribute>
     </Attributes>
     <Icon>
       OperatingSystem.svg
@@ -9,7 +9,8 @@
   </Operating_System>
   <Network>
     <Attributes>
-      IP Address,Range
+      <Attribute default="127.0.0.0" check="String" primary="True">IP Address</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Range</Attribute>
     </Attributes>
     <Icon>
       Network.svg
@@ -17,15 +18,23 @@
   </Network>
   <Website>
     <Attributes>
-      URL
+      <Attribute default="https://example.com" check="URL" primary="True">URL</Attribute>
     </Attributes>
     <Icon>
       Website.svg
     </Icon>
   </Website>
+  <Onion_Website>
+    <Attributes>
+      <Attribute default="http://exampleexampleexampleexampleexampleexampleexampleexample.onion" check="Onion" primary="True">Onion URL</Attribute>
+    </Attributes>
+    <Icon>
+      OnionWebsite.svg
+    </Icon>
+  </Onion_Website>
   <Domain>
     <Attributes>
-      Domain Name
+      <Attribute default="example.com" check="Domain" primary="True">Domain Name</Attribute>
     </Attributes>
     <Icon>
       Domain.svg
@@ -33,7 +42,7 @@
   </Domain>
   <IP_Address>
     <Attributes>
-      IP Address
+      <Attribute default="127.0.0.1" check="IPv4" primary="True">IP Address</Attribute>
     </Attributes>
     <Icon>
       IP_Address.svg
@@ -41,15 +50,16 @@
   </IP_Address>
   <IPv6_Address>
     <Attributes>
-      IPv6 Address
+      <Attribute default="::1" check="IPv6" primary="True">IPv6 Address</Attribute>
     </Attributes>
     <Icon>
       IPv6_Address.svg
     </Icon>
-</IPv6_Address>
-<Autonomous_System>
+  </IPv6_Address>
+  <Autonomous_System>
     <Attributes>
-      AS Number,ASN Cidr
+      <Attribute default="AS0" check="ASN" primary="True">AS Number</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">ASN Cidr</Attribute>
     </Attributes>
     <Icon>
       AutonomousSystem.svg
@@ -57,7 +67,7 @@
   </Autonomous_System>
   <Port>
     <Attributes>
-      Port
+      <Attribute default="https://example.com:443" check="String" primary="True">Port</Attribute>
     </Attributes>
     <Icon>
       Port.svg
@@ -65,23 +75,27 @@
   </Port>
   <MAC_Address>
     <Attributes>
-      MAC Address
+      <Attribute default="AA:AA:AA:AA:AA:AA" check="MAC" primary="True">MAC Address</Attribute>
     </Attributes>
     <Icon>
       MAC_Address.svg
     </Icon>
   </MAC_Address>
-  <Network>
-    <Attributes>
-      IP Address,Range
-    </Attributes>
-  </Network>
   <Website_Infrastructure>
     <Attributes>
-      Infrastructure
+      <Attribute default="Infrastructure" check="String" primary="True">Infrastructure</Attribute>
     </Attributes>
     <Icon>
       WebInfrastructure.svg
     </Icon>
   </Website_Infrastructure>
+  <Web_Certificate>
+    <Attributes>
+      <Attribute default="Subdomain" check="String" primary="True">Subdomain</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Authority</Attribute>
+    </Attributes>
+    <Icon>
+      Web_Certificate.svg
+    </Icon>
+  </Web_Certificate>
 </Infrastructure>

Core/Entities/Materials.xml

@@ -1,15 +1,26 @@
 <Materials>
   <Document>
     <Attributes>
-      Document Name,File Path
+      <Attribute default="Document" check="String" primary="True">Document Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
     </Attributes>
     <Icon>
       Document.svg
     </Icon>
   </Document>
+  <Spreadsheet>
+    <Attributes>
+      <Attribute default="Spreadsheet" check="String" primary="True">Spreadsheet Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
+    </Attributes>
+    <Icon>
+      Spreadsheet.svg
+    </Icon>
+  </Spreadsheet>
   <Image>
     <Attributes>
-      Image Name,File Path
+      <Attribute default="Image" check="String" primary="True">Image Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
     </Attributes>
     <Icon>
       Image.svg
@@ -17,7 +28,8 @@
   </Image>
   <Video>
     <Attributes>
-      Video Name,File Path
+      <Attribute default="Video" check="String" primary="True">Video Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
     </Attributes>
     <Icon>
       Video.svg
@@ -25,7 +37,8 @@
   </Video>
   <Archive>
     <Attributes>
-      Archive Name,File Path
+      <Attribute default="Archive" check="String" primary="True">Archive Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
     </Attributes>
     <Icon>
       Archive.svg
@@ -33,7 +46,8 @@
   </Archive>
   <Disk>
     <Attributes>
-      Disk Name,File Path
+      <Attribute default="Disk" check="String" primary="True">Disk Name</Attribute>
+      <Attribute default="DefaultFilePath" check="String" primary="False">File Path</Attribute>
     </Attributes>
     <Icon>
       HardDisk.svg

Core/Entities/Meta.xml

@@ -1,7 +1,8 @@
 <Meta>
   <EntityGroup>
     <Attributes>
-      Group Name,Child UIDs
+      <Attribute default="Entity Group" check="String" primary="True">Group Name</Attribute>
+      <Attribute default="[]" check="None" primary="False">Child UIDs</Attribute>
     </Attributes>
     <Icon>
       GroupNode.svg

Core/Entities/Places.xml

@@ -1,7 +1,9 @@
 <Places>
   <GeoCoordinates>
     <Attributes>
-      Label,Latitude,Longitude
+      <Attribute default="Location Label" check="String" primary="True">Label</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Latitude</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Longitude</Attribute>
     </Attributes>
     <Icon>
       Geolocation.svg
@@ -9,7 +11,10 @@
   </GeoCoordinates>
   <Address>
     <Attributes>
-      Street Address,Locality,Postal Code,Country
+      <Attribute default="Street Address" check="String" primary="True">Street Address</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Locality</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Postal Code</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Country</Attribute>
     </Attributes>
     <Icon>
       Address.svg
@@ -17,7 +22,7 @@
   </Address>
   <Country>
     <Attributes>
-      Country Name
+      <Attribute default="Country Name" check="String" primary="True">Country Name</Attribute>
     </Attributes>
     <Icon>
       Country.svg
@@ -25,7 +30,7 @@
   </Country>
   <City>
     <Attributes>
-      City Name
+      <Attribute default="City Name" check="String" primary="True">City Name</Attribute>
     </Attributes>
     <Icon>
       City.svg
@@ -33,7 +38,10 @@
   </City>
   <Circular_Area>
     <Attributes>
-      Label,Latitude,Longitude,Radius
+      <Attribute default="Area Label" check="String" primary="True">Label</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Latitude</Attribute>
+      <Attribute default="0.0" check="Float" primary="False">Longitude</Attribute>
+      <Attribute default="Unknown" check="String" primary="False">Radius</Attribute>
     </Attributes>
     <Icon>
       GeoArea.svg

Core/Entities/Social Media.xml

@@ -1,7 +1,7 @@
 <Social>
     <Social_Media_Group>
         <Attributes>
-            Group Name,Social Media Site
+            <Attribute default="Guest Group" check="String" primary="True">Group Name</Attribute>
         </Attributes>
         <Icon>
             Social_Media_Group.svg
@@ -9,7 +9,7 @@
     </Social_Media_Group>
     <Social_Media_Handle>
         <Attributes>
-            User Name
+            <Attribute default="Guest123" check="String" primary="True">User Name</Attribute>
         </Attributes>
         <Icon>
             Social_Media_Account.svg

Core/Entities/Software.xml

@@ -1,7 +1,7 @@
 <Software>
     <Application>
         <Attributes>
-            Application Name
+            <Attribute default="Application Name" check="String" primary="True">Application Name</Attribute>
         </Attributes>
         <Icon>
             Software.svg
@@ -9,10 +9,20 @@
     </Application>
     <Hash>
         <Attributes>
-            Hash Value,Hash Algorithm
+            <Attribute default="Hash Digest Value" check="String" primary="True">Hash Value</Attribute>
+            <Attribute default="Unknown" check="String" primary="False">Hash Algorithm</Attribute>
         </Attributes>
         <Icon>
             Hash.svg
         </Icon>
     </Hash>
+    <CVE>
+        <Attributes>
+            <Attribute default="Unassigned CVE" check="String" primary="True">CVE</Attribute>
+            <Attribute default="0.0" check="Float" primary="False">CVSS Score</Attribute>
+        </Attributes>
+        <Icon>
+            CVE.svg
+        </Icon>
+    </CVE>
 </Software>

Core/Entities/Vehicles.xml

@@ -0,0 +1,24 @@
+<Vehicles>
+    <Vessel>
+        <Attributes>
+            <Attribute default="IMO 0" check="String" primary="True">Vessel Registration</Attribute>
+            <Attribute default="Vessel" check="String" primary="False">Vessel Name</Attribute>
+            <Attribute default="000000" check="String" primary="False">Vessel CallSign</Attribute>
+            <Attribute default="XX" check="String" primary="False">Vessel Flag</Attribute>
+        </Attributes>
+        <Icon>
+            Vessel.svg
+        </Icon>
+    </Vessel>
+    <Aircraft>
+        <Attributes>
+            <Attribute default="0000" check="String" primary="True">Aircraft Serial Number</Attribute>
+            <Attribute default="00-000" check="String" primary="False">Aircraft Registration</Attribute>
+            <Attribute default="Aircraft" check="String" primary="False">Aircraft Name</Attribute>
+            <Attribute default="Aircraft Model" check="String" primary="False">Aircraft Model</Attribute>
+        </Attributes>
+        <Icon>
+            Aircraft.svg
+        </Icon>
+    </Aircraft>
+</Vehicles>

Core/EntityDB.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
 from shutil import move
-from pickle import load, dump
+from msgpack import load, dump
 from threading import Lock
 from pathlib import Path
+from typing import Union
 
 import networkx as nx
 
@@ -14,9 +15,7 @@ class EntitiesDB:
     links on a project-wide scale.
     """
 
-    def __init__(self, mainWindow, messageHandler, resourceHandler):
-        self.messageHandler = messageHandler
-        self.resourceHandler = resourceHandler
+    def __init__(self, mainWindow) -> None:
         self.mainWindow = mainWindow
         self.dbLock = Lock()
         self.database = None
@@ -24,93 +23,86 @@ class EntitiesDB:
         self.loadDatabase()
         self.resetTimeline()
 
-    def loadDatabase(self):
+    def loadDatabase(self) -> None:
         """
-        Load DiGraph from pickle file.
+        Load DiGraph from LinkScope Database file - msgpack dumped object.
         """
-        self.dbLock.acquire()
-        if self.database is not None:
-            self.save()
-        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.pkl")
-        self.messageHandler.debug('Opening Database at: ' + str(databaseFile))
-        try:
-            dbFile = open(databaseFile, "rb")
-            self.database = load(dbFile)
-            dbFile.close()
-            self.messageHandler.info('Loaded Local Entities Database.')
-        except FileNotFoundError:
-            self.messageHandler.info('Creating new Local Entities Database.')
-            self.database = nx.DiGraph()
-        finally:
-            self.dbLock.release()
+        with self.dbLock:
+            if self.database is not None:
+                self.save()
+            databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.lsdb")
+            self.mainWindow.MESSAGEHANDLER.debug(f'Opening Database at: {databaseFile}')
+            try:
+                with open(databaseFile, "rb") as dbFile:
+                    self.database = self.mainWindow.RESOURCEHANDLER.reconstructGraphFullFromFile(load(dbFile))
+                self.mainWindow.MESSAGEHANDLER.info('Loaded Local Entities Database.')
+            except FileNotFoundError:
+                self.mainWindow.MESSAGEHANDLER.info('Creating new Local Entities Database.')
+                self.database = nx.DiGraph()
+            except Exception as exc:
+                self.mainWindow.MESSAGEHANDLER.error(
+                    f'Cannot parse Database: {exc}\nCreating new Local Entities Database.',
+                    popUp=True)
+                self.database = nx.DiGraph()
 
-    def resetTimeline(self):
+    def resetTimeline(self) -> None:
         """
-        Reset the timeline on dockBarThree to reflect the current state of the entities database.
+        Reset the timeline on dockBarThree to reflect the current state of the database.
         """
-        self.dbLock.acquire()
-        if self.database is not None:
-            self.mainWindow.resetTimeline(self.database)
-        self.dbLock.release()
+        with self.dbLock:
+            if self.database is not None:
+                self.mainWindow.resetTimeline(self.database)
 
-    def updateTimeline(self, node, added: bool, updateGraph: bool = True):
+    def updateTimeline(self, node, added: bool, updateGraph: bool = True) -> None:
         """
-        Update the timeline on dockBarThree to reflect the newest change of the entities database.
+        Update the timeline on dockBarThree to reflect the newest change of the database.
         """
-        self.dbLock.acquire()
-        self.mainWindow.updateTimeline(node, added, updateGraph)
-        self.dbLock.release()
+        with self.dbLock:
+            self.mainWindow.updateTimeline(node, added, updateGraph)
 
-    def save(self):
+    def save(self) -> None:
         """
-        Saves the graph (pickles it too) to the specified file.
+        Saves the graph to the specified file.
         """
 
         # Get the database file path again, in case it changed.
-        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.pkl")
+        databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.lsdb")
         if databaseFile is None:
             raise ValueError('Database File is None, cannot save database.')
-        self.dbLock.acquire()
-        tmpSavePath = databaseFile.with_suffix(databaseFile.suffix + '.tmp')
-        dbFile = open(tmpSavePath, "wb")
-        dump(self.database, dbFile)
-        dbFile.close()
-        move(tmpSavePath, databaseFile)
-        self.messageHandler.info('Database Saved.')
-        self.dbLock.release()
+        with self.dbLock:
+            tmpSavePath = databaseFile.with_suffix(f'{databaseFile.suffix}.tmp')
+            with open(tmpSavePath, "wb") as dbFile:
+                dump(self.mainWindow.RESOURCEHANDLER.deconstructGraphForFileDump(self.database), dbFile)
+            move(tmpSavePath, databaseFile)
+            self.mainWindow.MESSAGEHANDLER.info('Database Saved.')
 
-    def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True):
+    def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True) -> Union[dict, None]:
         """
         Adds the entity represented by the json dictionary to the database.
-        :param updateTimeline:
-        :param entJson:
-        :param fromServer:
-        :return:
         """
-        self.dbLock.acquire()
-        returnValue = None
+        with self.dbLock:
+            returnValue = None
 
-        # Check if we're overwriting an existing entity
-        exists = None
-        if entJson.get('uid') is not None:
-            exists = self.getEntityNoLock(entJson.get('uid'))
+            # Check if we're overwriting an existing entity
+            exists = None
+            if entJson.get('uid') is not None:
+                exists = self.getEntityNoLock(entJson.get('uid'))
 
-        entity = self.resourceHandler.getEntityJson(
-            entJson.get('Entity Type'),
-            entJson)
+            entity = self.mainWindow.RESOURCEHANDLER.getEntityJson(
+                entJson.get('Entity Type'),
+                entJson)
+
+            if entity is None:
+                return returnValue
+            # Use uid as key. Code is holdover from the time when primary field == uid.
+            self.database.add_node(entity['uid'], **entity)
+            returnValue = entity
+            if exists:
+                # Update canvases if the node already exists.
+                self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
 
-        if entity is None:
-            self.dbLock.release()
-            return returnValue
-        # Use uid as key. Code is holdover from time where primary field == uid.
-        self.database.add_node(entity['uid'], **entity)
-        returnValue = entity
-        if exists:
-            # Update canvases if the node already exists.
-            self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
-        self.dbLock.release()
         if not fromServer:
-            self.mainWindow.sendLocalDatabaseUpdateToServer(entity, True)
+            self.mainWindow.sendLocalDatabaseUpdateToServer(entity, 1)
         self.mainWindow.populateEntitiesWidget(returnValue, add=True)
 
         if updateTimeline:
@@ -121,85 +113,126 @@ class EntitiesDB:
 
         return returnValue
 
-    def addLink(self, linkJson: dict, fromServer=False):
+    def addEntities(self, entitiesJsonList: Union[list, set, tuple], fromServer: bool = False) -> list:
+        with self.dbLock:
+            returnValue = []
+
+            for entJson in entitiesJsonList:
+                # Check if we're overwriting an existing entity
+                exists = None
+                if entJson.get('uid') is not None:
+                    exists = self.getEntityNoLock(entJson.get('uid'))
+
+                entity = self.mainWindow.RESOURCEHANDLER.getEntityJson(
+                    entJson.get('Entity Type'),
+                    entJson)
+
+                if entity is None:
+                    continue
+                # Use uid as key. Code is holdover from the time when primary field == uid.
+                self.database.add_node(entity['uid'], **entity)
+                returnValue.append(entity)
+                if exists:
+                    # Update canvases if the node already exists.
+                    self.mainWindow.updateEntityNodeLabelsOnCanvases(entity['uid'], entity[list(entity)[1]])
+                if not fromServer:
+                    self.mainWindow.sendLocalDatabaseUpdateToServer(entity, 1)
+                self.mainWindow.populateEntitiesWidget(entity, add=True)
+
+        self.resetTimeline()
+
+        return returnValue
+
+    def addLink(self, linkJson: dict, fromServer: bool = False, overwrite: bool = False) -> Union[dict, None]:
         """
         Add a link between two entities in the database.
 
+        :param overwrite:
         :param linkJson:
         :param fromServer:
         :return:
         """
-        self.dbLock.acquire()
-        exists = self.isLinkNoLock(linkJson['uid'])
-        link = self.resourceHandler.getLinkJson(linkJson)
-        if link is None:
-            # This can technically be caused by a race condition if the user
-            #   either tries really hard or gets really unlucky.
-            # Caused by deleting a node faster than the link can be created.
-            self.messageHandler.error("Attempted to add Link with "
-                                      "no uid to database.")
-        else:
-            linkUID = link['uid']
-            self.database.add_edge(linkUID[0], linkUID[1], **link)
-            if exists:
-                # Update canvases if the link already exists.
-                self.mainWindow.updateLinkLabelsOnCanvases(linkUID[0] + linkUID[1], link['Resolution'])
-        self.dbLock.release()
+        with self.dbLock:
+            exists = self.isLinkNoLock(linkJson['uid'])
+            link = self.mainWindow.RESOURCEHANDLER.getLinkJson(linkJson)
+            if link is None:
+                # This can technically be caused by a race condition if the user
+                #   either tries really hard or gets really unlucky.
+                # Caused by deleting a node faster than the link can be created.
+                self.mainWindow.MESSAGEHANDLER.error("Attempted to add Link with no uid to database.", popUp=True)
+                return None
+            else:
+                linkUID = link['uid']
+                if exists:
+                    newRes = link.get('Resolution')
+                    newNotes = link.get('Notes')
+                    if newRes and newRes != exists['Resolution']:
+                        if overwrite:
+                            link['Resolution'] = newRes
+                        else:
+                            link['Resolution'] = f"{exists['Resolution']} | {newRes}"
+                    if newNotes and newNotes != exists['Notes'] and newNotes != 'None':
+                        if overwrite:
+                            link['Notes'] = str(newNotes)
+                        else:
+                            link['Notes'] = f"{exists['Notes']}\n\n{str(newNotes)}"
+                    exists.update(link)
+                    link.update(exists)
+                    # Update canvases if the link already exists.
+                    # We can do this before updating the database here because the GUI will be updated only after this
+                    #   function returns. If we ever execute this function outside the main event loop, we will need
+                    #   to alter the execution flow.
+                    self.mainWindow.updateLinkLabelsOnCanvases(f"{linkUID[0]}{linkUID[1]}", link['Resolution'])
+                self.database.add_edge(linkUID[0], linkUID[1], **link)
+
         if not fromServer:
-            self.mainWindow.sendLocalDatabaseUpdateToServer(link, True)
+            if overwrite:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(link, 3)
+            else:
+                self.mainWindow.sendLocalDatabaseUpdateToServer(link, 1)
         return link
 
-    def getEntity(self, uid: str):
+    def getEntity(self, uid: str) -> Union[dict, None]:
         """
         Returns the attributes of the given entity uid as a dict.
         """
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = self.database.nodes[uid]
-        except KeyError:
-            self.messageHandler.warning(
-                "Tried to get entity with nonexistent UID: " + uid)
-        finally:
-            self.dbLock.release()
-            return returnValue
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.nodes[uid]
+            except KeyError:
+                self.mainWindow.MESSAGEHANDLER.warning(f"Tried to get entity with nonexistent UID: {uid}")
+            finally:
+                return returnValue
 
-    def getAllEntities(self):
+    def getAllEntities(self) -> Union[None, list]:
         """
         Returns a list containing the Json representation of every entity in the database.
         """
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = []
-            for node in self.database.nodes():
-                returnValue += [self.database.nodes[node]]
-        except KeyError:
-            self.messageHandler.error(
-                "Tried to get entity with nonexistent UID.")
-        finally:
-            self.dbLock.release()
-            return returnValue
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = [self.database.nodes[node] for node in self.database.nodes()]
+            except KeyError as keyError:
+                self.mainWindow.MESSAGEHANDLER.error(f"Tried to get entity with nonexistent UID. Error: {keyError}")
+            finally:
+                return returnValue
 
-    def getAllLinks(self):
+    def getAllLinks(self) -> Union[None, list]:
         """
         Returns a list containing the Json representation of every link in the database.
         :return:
         """
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = []
-            for edge in self.database.edges():
-                returnValue += [self.database.edges[edge]]
-        except KeyError:
-            self.messageHandler.error(
-                "Tried to get link with nonexistent UID.")
-        finally:
-            self.dbLock.release()
-            return returnValue
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = [self.database.edges[edge] for edge in self.database.edges()]
+            except KeyError:
+                self.mainWindow.MESSAGEHANDLER.error("Tried to get link with nonexistent UID.")
+            finally:
+                return returnValue
 
-    def getEntityNoLock(self, uid: str):
+    def getEntityNoLock(self, uid: str) -> Union[None, dict]:
         """
         Returns the attributes of the given entity uid as a dict.
 
@@ -214,202 +247,181 @@ class EntitiesDB:
         finally:
             return returnValue
 
-    def getLink(self, uid):
+    def getLink(self, uid) -> Union[None, dict]:
         """
         Returns the attributes of the given link uid as a dict.
         """
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = self.database.edges[uid]
-        except KeyError:
-            self.messageHandler.error(
-                "Tried to get link with nonexistent UID.")
-        finally:
-            self.dbLock.release()
-            return returnValue
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.edges[uid]
+            except KeyError:
+                self.mainWindow.MESSAGEHANDLER.error(f"Tried to get link with nonexistent UID: {uid}")
+            finally:
+                return returnValue
 
-    def removeEntity(self, uid: str, fromServer=False):
+    def removeEntity(self, uid: str, fromServer=False, updateTimeLine=True) -> None:
         """
-        Removes the entity with the given uid, if it
-        exists.
+        Removes the entity with the given uid, if it exists.
         """
-        self.dbLock.acquire()
-        ent = None
-        if self.isNodeNoLock(uid):
-            ent = self.getEntityNoLock(uid)
-            self.mainWindow.populateEntitiesWidget(ent, add=False)
-            self.database.remove_node(uid)
-        self.dbLock.release()
+        with self.dbLock:
+            ent = None
+            if self.isNodeNoLock(uid):
+                ent = self.getEntityNoLock(uid)
+                self.mainWindow.populateEntitiesWidget(ent, add=False)
+                self.database.remove_node(uid)
+
         if ent is not None:
             self.mainWindow.handleGroupNodeUpdateAfterEntityDeletion(uid)  # Blocking - locks the db.
             if not fromServer:
-                self.mainWindow.sendLocalDatabaseUpdateToServer(ent, False)
-            self.updateTimeline(ent, False)
+                self.mainWindow.sendLocalDatabaseUpdateToServer(ent, 2)
+            if updateTimeLine:
+                self.updateTimeline(ent, False)
 
-    def removeLink(self, uid, fromServer=False):
+    def removeLink(self, uid, fromServer=False) -> None:
         """
         Removes the link with the given uid (in string or tuple form),
         if it exists.
         """
-        self.dbLock.acquire()
-        if self.isLinkNoLock(uid):
-            self.database.remove_edge(uid[0], uid[1])
-        self.dbLock.release()
+        with self.dbLock:
+            if self.isLinkNoLock(uid):
+                self.database.remove_edge(uid[0], uid[1])
         if not fromServer:
-            self.mainWindow.sendLocalDatabaseUpdateToServer({"uid": uid}, False)
+            self.mainWindow.sendLocalDatabaseUpdateToServer({"uid": uid}, 2)
 
-    def doesEntityExist(self, primaryAttr: str):
+    def doesEntityExist(self, primaryAttr: str) -> bool:
         """
         Checks if an entity with the specified primary attribute exists.
         """
-        self.dbLock.acquire()
-        result = False
-        for node in self.database.nodes():
-            details = self.database.nodes[node]
-            if details[list(details)[1]] == primaryAttr:
-                result = True
-                break
-        self.dbLock.release()
+        with self.dbLock:
+            result = False
+            for node in self.database.nodes():
+                details = self.database.nodes[node]
+                if details[list(details)[1]] == primaryAttr:
+                    result = True
+                    break
         return result
 
-    def getLinkIfExists(self, uid):
+    def getEntityOfType(self, primaryAttr: str, entityType: str) -> Union[dict, None]:
+        """
+        Checks if an entity with the specified primary attribute exists, and if it does, return it.
+        """
+        result = None
+        primaryField = self.mainWindow.RESOURCEHANDLER.getPrimaryFieldForEntityType(entityType)
+        if primaryField is None:
+            return result
+        with self.dbLock:
+            for node in self.database.nodes():
+                details = self.database.nodes[node]
+                if details['Entity Type'] == entityType and details[primaryField] == primaryAttr:
+                    result = dict(details)
+                    break
+        return result
+
+    def getLinkIfExists(self, uid) -> Union[None, dict]:
         """
         Returns the attributes of the given link uid as a dict.
         Does not create an error if the link does not exist.
         """
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = self.database.edges[uid]
-        except KeyError:
-            pass
-        finally:
-            self.dbLock.release()
-            return returnValue
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.database.edges[uid]
+            except KeyError:
+                pass
+            finally:
+                return returnValue
 
     def getIncomingLinks(self, uid: str):
         """
         Get all incoming edges for the given entity uid (primary attribute).
         """
-        self.dbLock.acquire()
-        returnValue = None
-        if self.isNodeNoLock(uid):
-            returnValue = self.database.in_edges(uid)
-        self.dbLock.release()
+        with self.dbLock:
+            returnValue = self.database.in_edges(uid) if self.isNodeNoLock(uid) else None
         return returnValue
 
     def getOutgoingLinks(self, uid: str):
         """
         Get all outgoing edges for the given entity uid (primary attribute).
         """
-        self.dbLock.acquire()
-        returnValue = None
-        if self.isNodeNoLock(uid):
-            returnValue = self.database.out_edges(uid)
-        self.dbLock.release()
+        with self.dbLock:
+            returnValue = self.database.out_edges(uid) if self.isNodeNoLock(uid) else None
         return returnValue
 
-    def isNode(self, uid: str):
+    def isNode(self, uid: Union[str, list, tuple]) -> bool:
         """
         Returns True if the uid (primary attribute) given exists as
         an entity, and False otherwise.
         """
-        self.dbLock.acquire()
-        returnValue = False
-        if self.database.nodes.get(uid) is not None:
-            returnValue = True
-        self.dbLock.release()
+        with self.dbLock:
+            returnValue = isinstance(uid, str) and self.database.nodes.get(uid) is not None
         return returnValue
 
-    def isNodeNoLock(self, uid: str):
+    def isNodeNoLock(self, uid: str) -> bool:
         """
         Returns True if the uid (primary attribute) given exists as
         an entity, and False otherwise.
 
         Used only in this class, as it does not lock.
         """
-        if self.database.nodes.get(uid) is not None:
-            return True
-        return False
+        return self.database.nodes.get(uid) is not None
 
-    def isLink(self, uid: tuple):
+    def isLink(self, uid: Union[str, list, tuple]) -> bool:
         """
         Returns True if the uid given exists as a link, and False otherwise.
         """
-        self.dbLock.acquire()
-        returnValue = False
-        item = self.database.edges.get(uid)
-        if item is not None:
-            returnValue = True
-        self.dbLock.release()
+        with self.dbLock:
+            returnValue = isinstance(uid, tuple) and self.database.edges.get(uid) is not None
         return returnValue
 
-    def isLinkNoLock(self, uid: tuple):
+    def isLinkNoLock(self, uid: tuple) -> Union[bool, dict]:
         """
         Returns True if the uid given exists as a link, and False otherwise.
         
         Used only in this class, as it does not lock.
         """
         if self.database.edges.get(uid) is not None:
-            return True
+            return self.database.edges[uid]
         return False
 
-    def getEntityType(self, uid: str):
-        self.dbLock.acquire()
-        returnValue = None
-        try:
-            returnValue = self.getEntityNoLock(uid)['Entity Type']
-        except KeyError:
-            pass
-        finally:
-            self.dbLock.release()
-            return returnValue
+    def getEntityType(self, uid: str) -> Union[None, dict]:
+        with self.dbLock:
+            returnValue = None
+            try:
+                returnValue = self.getEntityNoLock(uid)['Entity Type']
+            except KeyError:
+                pass
+            finally:
+                return returnValue
 
-    def mergeDatabases(self, newDB_nodes: dict, newDB_edges: dict, fromServer=True):
+    def mergeDatabases(self, newDB_nodes: dict, newDB_edges: dict, fromServer=True) -> None:
         """
         Merges the existing database with the one provided.
-        
-        Overwrites older attributes with newer ones.
+        Overwrites older attributes with newer ones based on date last edited.
         """
-        self.dbLock.acquire()
-        differenceGraph = nx.DiGraph()
-        # Note: If we ever receive a node without a 'Date Last Edited' field, ignore it.
-        differenceGraph.add_nodes_from([(x, newDB_nodes[x])
-                                        for x in newDB_nodes if (x not in self.database.nodes()) or
-                                        (
-                                                x in self.database.nodes() and
-                                                newDB_nodes[x].get('Date Last Edited') and
-                                                newDB_nodes[x]['Date Last Edited'] >
-                                                self.database.nodes[x]['Date Last Edited']
-                                        )
-                                        ])
-        differenceGraph.add_edges_from([(x, y, newDB_edges[(x, y)])
-                                        for x, y in newDB_edges if ((x, y) not in self.database.edges()) or
-                                        (
-                                                (x, y) in self.database.edges() and
-                                                newDB_edges[(x, y)].get('Date Last Edited') and
-                                                newDB_edges[(x, y)]['Date Last Edited'] >
-                                                self.database.edges[(x, y)]['Date Last Edited']
-                                        )
-                                        ])
-        if differenceGraph.number_of_nodes() > 0:
-            self.database = nx.compose(self.database, differenceGraph)
-            # Some nodes given by differenceGraph may be empty dicts, with an existing node's uid as the key.
-            for node in differenceGraph.nodes:
-                self.mainWindow.populateEntitiesWidget(self.database.nodes[node], add=True)
-
-            if not fromServer:
-                if self.mainWindow.FCOM.isConnected():
-                    # diffNew = nx.DiGraph()
-                    # diffNew.add_nodes_from([(x, self.database.nodes[x])
-                    #                        for x in self.database.nodes() if x not in differenceGraph.nodes()
-                    #                        ])
-                    # diffNew.add_edges_from([(x, y, self.database.edges[(x, y)])
-                    #                        for x, y in self.database.edges() if (x, y) not in differenceGraph.edges()
-                    #                        ])
+        with self.dbLock:
+            differenceGraph = nx.DiGraph()
+            differenceGraph.add_nodes_from([(n, nDict)
+                                            for n, nDict in newDB_nodes.items() if (n not in self.database.nodes()) or
+                                            (
+                                                    nDict.get('Date Last Edited', '') >
+                                                    self.database.nodes[n].get('Date Last Edited', '')
+                                            )
+                                            ])
+            differenceGraph.add_edges_from([(e[0], e[1], eDict)
+                                            for e, eDict in newDB_edges.items() if (e not in self.database.edges()) or
+                                            (
+                                                    eDict.get('Date Last Edited', '') >
+                                                    self.database.edges[e].get('Date Last Edited', '')
+                                            )
+                                            ])
+            if differenceGraph.number_of_nodes():
+                self.database = nx.compose(self.database, differenceGraph)
+                # Some nodes given by differenceGraph may be empty dicts, with an existing node's uid as the key.
+                for node in differenceGraph.nodes:
+                    self.mainWindow.populateEntitiesWidget(self.database.nodes[node], add=True)
 
+                if not fromServer and self.mainWindow.FCOM.isConnected():
+                    # Assume we are already synced with server, so just send the difference.
                     self.mainWindow.FCOM.syncDatabase(self.mainWindow.SETTINGS.value("Project/Server/Project"),
                                                       differenceGraph)
-
-        self.dbLock.release()

Core/FrontendCommunicationsHandler.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 
+import contextlib
 from ast import literal_eval
 from typing import Union
 from pathlib import Path
@@ -25,6 +26,12 @@ from cryptography.hazmat.primitives.serialization import load_der_public_key
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.exceptions import InvalidTag
 
+# Amount of data to place in each message
+MESSAGE_DATA_SIZE = 8192 * 5
+
+# Needs to be a bit bigger than MESSAGE_DATA_SIZE
+RECV_SIZE = MESSAGE_DATA_SIZE + 1024
+
 # All functions use this to run. Checks if the program is closing.
 closeSoftwareLock = threading.Lock()
 closeSoftware = False
@@ -43,18 +50,22 @@ class CommunicationsHandler(QtCore.QObject):
     connected_to_server_listener = QtCore.Signal(str)
     receive_question_answer = QtCore.Signal(dict)
     receive_chat_message = QtCore.Signal(str)
+    receive_collectors_signal = QtCore.Signal(dict, dict)
+    receive_start_collector_signal = QtCore.Signal(str, str, str, list, dict)
+    receive_collector_result_signal = QtCore.Signal(str, str, str, list)
     receive_resolutions_signal = QtCore.Signal(dict)
-    receive_completed_resolution_result_signal = QtCore.Signal(str, list)
-    receive_completed_resolution_string_result_signal = QtCore.Signal(str, str)
+    receive_completed_resolution_result_signal = QtCore.Signal(str, list, str)
+    receive_completed_resolution_string_result_signal = QtCore.Signal(str, str, str)
     receive_document_summary_signal = QtCore.Signal(str, str)
     remove_server_resolution_from_running_signal = QtCore.Signal(str)
     receive_projects_list_signal = QtCore.Signal(list)
+    delete_server_project_signal = QtCore.Signal(str)
     receive_project_canvases_list_signal = QtCore.Signal(list)
     open_project_signal = QtCore.Signal(str)
     close_project_signal = QtCore.Signal()
     open_project_canvas_signal = QtCore.Signal(str)
     close_project_canvas_signal = QtCore.Signal(str)
-    receive_project_database_update = QtCore.Signal(dict, bool)
+    receive_project_database_update = QtCore.Signal(dict, int)
     receive_project_canvas_update_node = QtCore.Signal(str, str)
     receive_project_canvas_update_link = QtCore.Signal(str, tuple)
     receive_sync_database = QtCore.Signal(dict, dict)
@@ -77,16 +88,20 @@ class CommunicationsHandler(QtCore.QObject):
         self.connected_to_server_listener.connect(self.mainWindow.connectedToServerListener)
         self.receive_question_answer.connect(self.mainWindow.questionAnswerListener)
         self.receive_chat_message.connect(self.mainWindow.receiveChatMessage)
+        self.receive_collectors_signal.connect(self.mainWindow.addCollectorsFromServerListener)
+        self.receive_start_collector_signal.connect(self.mainWindow.startNewCollectorListener)
+        self.receive_collector_result_signal.connect(self.mainWindow.receiveCollectorResultListener)
         self.receive_resolutions_signal.connect(self.mainWindow.addResolutionsFromServerListener)
         self.receive_completed_resolution_result_signal.connect(self.mainWindow.resolutionSignalListener)
         self.receive_completed_resolution_string_result_signal.connect(self.mainWindow.resolutionSignalListener)
         self.receive_document_summary_signal.connect(self.mainWindow.receiveSummaryOfDocument)
         self.remove_server_resolution_from_running_signal.connect(self.mainWindow.cleanServerResolutionListener)
+        self.delete_server_project_signal.connect(self.mainWindow.receiveProjectDeleteListener)
         self.receive_projects_list_signal.connect(self.mainWindow.receiveProjectsListListener)
         self.receive_project_canvases_list_signal.connect(self.mainWindow.receiveProjectCanvasesListListener)
         self.open_project_signal.connect(self.mainWindow.openServerProjectListener)
         self.open_project_canvas_signal.connect(self.mainWindow.openServerCanvasListener)
-        self.close_project_signal.connect(self.mainWindow.closeCurrentServerProject)
+        self.close_project_signal.connect(self.mainWindow.closeServerProjectListener)
         self.close_project_canvas_signal.connect(self.mainWindow.closeServerCanvasListener)
         self.receive_project_database_update.connect(self.mainWindow.receiveServerDatabaseUpdate)
         self.receive_project_canvas_update_node.connect(self.mainWindow.receiveServerCanvasUpdate)
@@ -103,7 +118,7 @@ class CommunicationsHandler(QtCore.QObject):
         self.threadInb = None
         self.sock = None
 
-    def beginCommunications(self, password: str, server: str, port: int = 3777):
+    def beginCommunications(self, password: str, server: str, port: int = 3777) -> bool:
         global closeSoftwareLock
         global closeSoftware
         with closeSoftwareLock:  # Not strictly necessary, but might as well just in case.
@@ -116,7 +131,7 @@ class CommunicationsHandler(QtCore.QObject):
             self.sock.send(private_key.public_key().public_bytes(
                 Encoding.DER,
                 format=PublicFormat.SubjectPublicKeyInfo))
-            peer_public = load_der_public_key(self.sock.recv(5120))
+            peer_public = load_der_public_key(self.sock.recv(RECV_SIZE))
             shared_key = private_key.exchange(ec.ECDH(), peer_public)
             derived_key = HKDF(
                 algorithm=hashes.SHA3_256(),
@@ -132,14 +147,12 @@ class CommunicationsHandler(QtCore.QObject):
             passPad = 16 - (len(password) % 16)
             passMessage = encryptor.update(password.encode() + b"*" * passPad) + encryptor.finalize()
             self.sock.send(passMessage)
-            messageReceived = decrypter.update(self.sock.recv(5120)) + decrypter.finalize()
+            messageReceived = decrypter.update(self.sock.recv(RECV_SIZE)) + decrypter.finalize()
             if messageReceived == b"Passphrase is OK":
-                self.threadInc = threading.Thread(target=self.scanIncoming)
-                self.threadInc.setDaemon(True)
+                self.threadInc = threading.Thread(target=self.scanIncoming, daemon=True)
                 self.threadInc.start()
 
-                self.threadInb = threading.Thread(target=self.scanInbox)
-                self.threadInb.setDaemon(True)
+                self.threadInb = threading.Thread(target=self.scanInbox, daemon=True)
                 self.threadInb.start()
 
                 return True
@@ -148,29 +161,31 @@ class CommunicationsHandler(QtCore.QObject):
 
             self.mainWindow.MESSAGEHANDLER.warning("Server did not reply in the expected manner to "
                                                    "password authentication.", popUp=True)
+        except ConnectionRefusedError:
+            self.mainWindow.MESSAGEHANDLER.error("Did not connect: Server not running.", popUp=True, exc_info=False)
         except Exception as exception:
-            self.mainWindow.MESSAGEHANDLER.error("Did not connect: " + str(exception))
+            self.mainWindow.MESSAGEHANDLER.error(f"Did not connect: {str(exception)}")
 
         try:
             if self.sock is not None:
                 self.sock.shutdown(socket.SHUT_RDWR)
-                self.sock.close()
-                self.sock = None
         except OSError:
-            # This would typically occur if the socket is already closed.
-            self.mainWindow.MESSAGEHANDLER.info('Tried to close socket that was already closed.')
+            # This would typically occur if the socket is already shut down.
+            self.mainWindow.MESSAGEHANDLER.debug('Tried to shutdown socket that was already shut down.')
+        try:
+            self.sock.close()
+        finally:
+            self.sock = None
 
         return False
 
-    def isConnected(self):
+    def isConnected(self) -> bool:
         """
         Check if socket is in a working state.
         """
-        if self.sock is not None and self.sock.fileno() != -1:
-            return True
-        return False
+        return self.sock is not None and self.sock.fileno() != -1
 
-    def close(self):
+    def close(self) -> None:
         global closeSoftwareLock
         global closeSoftware
         with closeSoftwareLock:
@@ -187,79 +202,81 @@ class CommunicationsHandler(QtCore.QObject):
         self.downloadingFiles = {}
         self.uploadingFiles = {}
 
-    def encryptTransmission(self, bytesObject):
+    def encryptTransmission(self, bytesObject) -> bytes:
         encryptor = self.cipher.encryptor()
-        # No need to check for overflows or negative pads - transmitMessage practically ensures that the data
-        #   sent will always be less than or equal to 1280 bytes.
-        padNeeded = 1280 - len(bytesObject)
+        padNeeded = 16 - (len(bytesObject) % 16)
         message = encryptor.update(b'a' * padNeeded + bytesObject) + encryptor.finalize()
-        return message
+        return b64encode(message)
 
-    def decryptTransmission(self, bytesObject):
+    def decryptTransmission(self, bytesObject) -> Union[bytes, None]:
         decrypter = self.cipher.decryptor()
         try:
-            message = decrypter.update(bytesObject) + decrypter.finalize()
-            return message
+            return decrypter.update(bytesObject) + decrypter.finalize()
         except InvalidTag:
             # If the ciphertext cannot be decrypted to a valid message, return None.
             return None
 
-    def transmitMessage(self, messageJson: dict, showErrorOnBrokenPipe: bool = True):
-        # Note that Base64 encoded data is about 4/3 times the size of the original.
-        # 768 * 4/3 = 1024
-        print('Sending message:', messageJson)
-        argEncoded = b64encode(str(messageJson).encode())
+    def transmitMessage(self, messageJson: dict, showErrorOnBrokenPipe: bool = True) -> None:
+        self.mainWindow.MESSAGEHANDLER.debug(f'Sending Message: {messageJson}')
+        argEncoded = str(messageJson)
         largeMessageUUID = str(uuid4())
         try:
-            for data in range(0, len(argEncoded), 768):
-                partArg = argEncoded[data:data + 768]
-                done = data + 768 >= len(argEncoded)
+            for data in range(0, len(argEncoded), MESSAGE_DATA_SIZE):
+                partArg = argEncoded[data:data + MESSAGE_DATA_SIZE]
+                done = data + MESSAGE_DATA_SIZE >= len(argEncoded)
                 messageJson = {"uuid": largeMessageUUID,
-                               "message": partArg.decode(),
+                               "message": partArg,
                                "done": done}
-                self.sock.send(self.encryptTransmission(dumps(messageJson)))
+                self.sock.send(self.encryptTransmission(dumps(messageJson)) + b'\x03\x03\x03\x03\x03')
         except BrokenPipeError:
             if showErrorOnBrokenPipe:
                 self.mainWindow.MESSAGEHANDLER.error("Disconnected from server!", popUp=True, exc_info=False)
                 self.mainWindow.disconnectFromServer()
 
-    def closeSocket(self):
+    def closeSocket(self) -> None:
         """
         Called by the close function in this class when communications with the server are to be ended.
         :return:
         """
         try:
             self.transmitMessage({"Operation": "Close Socket", "Arguments": {}}, showErrorOnBrokenPipe=False)
-        except OSError:
-            # Typically this is due to bad file descriptor, i.e. server is closed.
-            pass
-        except AttributeError:
-            # This happens if no connection was established while the software was running
+        except (OSError, AttributeError):
+            # OSError thrown due to bad file descriptor, i.e. server is closed.
+            # AttributeError thrown if no connection was established while the software was running
             pass
         finally:
-            try:
+            with contextlib.suppress(OSError):
+                # OSError thrown if the socket is already closed.
                 if self.sock is not None:
                     self.sock.shutdown(socket.SHUT_RDWR)
-                    self.sock.close()
-                    self.sock = None
-            except OSError:
-                # This would typically occur if the socket is already closed.
-                pass
+            try:
+                self.sock.close()
+            finally:
+                self.sock = None
 
-    def scanIncoming(self):
+    def scanIncoming(self) -> None:
         """
         This function listens for incoming data, and puts it in the queue if
         it exists.
         """
         preInbox = {}
+        oldData = b''
         while True:
             try:
-                receivedInfo = self.sock.recv(5120)
+                receivedInfo = self.sock.recv(RECV_SIZE)
                 if receivedInfo == b'':
                     # Socket closed.
                     break
-                for message in range(0, len(receivedInfo), 1280):
-                    decryptedInfo = self.decryptTransmission(receivedInfo[message:message + 1280])
+                receivedInfo = oldData + receivedInfo
+                messages = receivedInfo.split(b'\x03\x03\x03\x03\x03\x03\x03\x03')
+                # Last message is either blank (i.e. '') or incomplete data, so we ignore it.
+                oldData = messages[-1]
+                messages = messages[:-1]
+                for message in messages:
+                    if len(message) == 0:
+                        continue
+                    message = b64decode(message)
+                    decryptedInfo = self.decryptTransmission(message)
                     if decryptedInfo is None:
                         self.mainWindow.MESSAGEHANDLER.warning("Invalid message received from server.")
                         continue
@@ -270,44 +287,82 @@ class CommunicationsHandler(QtCore.QObject):
                     else:
                         preInbox[messageID] = receivedMessage
                     if receivedMessage.get("done"):
-                        preInbox[messageID]["message"] = literal_eval(
-                            b64decode(preInbox[messageID]["message"]).decode())
+                        preInbox[messageID]["message"] = literal_eval(preInbox[messageID]["message"])
                         self.inbox.put(preInbox.pop(messageID).get("message"))
 
             except socket.error as socketError:
-                self.mainWindow.MESSAGEHANDLER.error('Socket Error: ' + str(socketError))
+                self.mainWindow.MESSAGEHANDLER.error(f'Socket Error: {str(socketError)}')
                 # If something happens, wait 2 seconds then try again.
-                closeSoftwareLock.acquire()
-                if not closeSoftware:
-                    closeSoftwareLock.release()
-                    time.sleep(2)
-                else:
-                    closeSoftwareLock.release()
-                    break
+                with closeSoftwareLock:
+                    if closeSoftware:
+                        break
+                time.sleep(2)
             except ValueError:
                 # E.g.: Unpack failed: incomplete input
-                pass
+                # In this case, we are being sent fragmented messages.
+                # They will be reconstructed eventually, once all the pieces get here.
+                continue
 
-    def askServerForResolutions(self):
+    def askServerForCollectors(self, continuing_collectors_dict: dict) -> None:
+        message = {"Operation": "Get Server Collectors",
+                   "Arguments": {
+                       'continuing_collectors_dict': continuing_collectors_dict
+                   }}
+        self.transmitMessage(message)
+
+    def receiveStartCollector(self, collector_category: str, collector_name: str, collector_uid: str,
+                              collector_entities: list, collector_parameters: dict):
+        self.receive_start_collector_signal.emit(collector_category, collector_name, collector_uid,
+                                                 collector_entities, collector_parameters)
+
+    def receiveCollectors(self, server_collectors: dict, continuing_collectors_info: dict) -> None:
+        self.receive_collectors_signal.emit(server_collectors, continuing_collectors_info)
+
+    def startServerCollector(self, collector_name: str, collector_entities: list, collector_parameters: dict,
+                             continueTimestamp: int = 0) -> None:
+        collector_entities_to_send = []
+        for entity in collector_entities:
+            with contextlib.suppress(KeyError):
+                dereferenced_entity = dict(entity)
+                collector_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any collector as of now: 2022/4/3.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+        message = {'Operation': 'Start Server Collector',
+                   'Arguments': {
+                       'collector_name': collector_name,
+                       'collector_entities': collector_entities_to_send,
+                       'collector_parameters': collector_parameters,
+                       'continue_time': continueTimestamp
+                   }}
+        self.transmitMessage(message)
+
+    def stopServerCollector(self, collector_uid: str) -> None:
+        message = {"Operation": "Stop Server Collector",
+                   "Arguments": {
+                       'collector_uid': collector_uid
+                   }}
+        self.transmitMessage(message)
+
+    def receiveCollectorResult(self, collector_name: str, collector_uid: str, timestamp: str, results: list) -> None:
+        self.receive_collector_result_signal.emit(collector_name, collector_uid, timestamp, results)
+
+    def askServerForResolutions(self) -> None:
         message = {"Operation": "Get Server Resolutions",
                    "Arguments": {}}
         self.transmitMessage(message)
 
-    def receiveResolutions(self, server_resolutions):
+    def receiveResolutions(self, server_resolutions) -> None:
         self.receive_resolutions_signal.emit(server_resolutions)
 
     def runRemoteResolution(self, resolution_name: str, resolution_entities: list, resolution_parameters: dict,
-                            resolution_uid: str):
+                            resolution_uid: str) -> None:
         resolution_entities_to_send = []
         for entity in resolution_entities:
-            try:
+            with contextlib.suppress(KeyError):
                 dereferenced_entity = dict(entity)
                 resolution_entities_to_send.append(dereferenced_entity)
-                # Icon is not necessary for any resolution as of now: 2022/1/2.
-                # Cutting it out saves data.
-                dereferenced_entity['Icon'] = ''
-            except KeyError:
-                pass
+                dereferenced_entity['Icon'] = dereferenced_entity['Icon'].toBase64().data()
         message = {'Operation': 'Run Resolution',
                    'Arguments': {
                        'resolution_name': resolution_name,
@@ -317,14 +372,20 @@ class CommunicationsHandler(QtCore.QObject):
                    }}
         self.transmitMessage(message)
 
-    def receiveResolutionResult(self, resolution_name: str, resolution_result: Union[list, str], resolution_uid: str):
+    def receiveResolutionResult(self, resolution_name: str, resolution_result: Union[list, str],
+                                resolution_uid: str) -> None:
         if isinstance(resolution_result, str):
-            self.receive_completed_resolution_string_result_signal.emit(resolution_name, resolution_result)
+            self.receive_completed_resolution_string_result_signal.emit(resolution_name, resolution_result,
+                                                                        resolution_uid)
         else:
-            self.receive_completed_resolution_result_signal.emit(resolution_name, resolution_result)
+            for res_result in resolution_result:
+                if res_icon := res_result[0].get('Icon'):
+                    res_result[0]['Icon'] = QtCore.QByteArray(b64decode(res_icon))
+            self.receive_completed_resolution_result_signal.emit(resolution_name, resolution_result,
+                                                                 resolution_uid)
         self.remove_server_resolution_from_running_signal.emit(resolution_uid)
 
-    def abortResolution(self, resolution_name: str, resolution_uid: str):
+    def abortResolution(self, resolution_name: str, resolution_uid: str) -> None:
         message = {'Operation': 'Abort Resolution',
                    'Arguments': {
                        'resolution_name': resolution_name,
@@ -332,42 +393,43 @@ class CommunicationsHandler(QtCore.QObject):
                    }}
         self.transmitMessage(message)
 
-    def askProjectsList(self):
+    def askProjectsList(self) -> None:
         message = {'Operation': 'Get Projects List',
                    'Arguments': {}}
         self.transmitMessage(message)
 
-    def receiveProjectsList(self, projects: list):
+    def receiveProjectsList(self, projects: list) -> None:
         self.receive_projects_list_signal.emit(projects)
 
-    def createProject(self, projectName: str, projectPassword: str):
+    def createProject(self, projectName: str, projectPassword: str) -> None:
         if self.isConnected():
             message = {'Operation': 'Create Project',
                        'Arguments': {'project_name': projectName, 'password': projectPassword}}
             self.transmitMessage(message)
 
-    def openProject(self, project_name: str, projectPassword: str):
+    def openProject(self, project_name: str, projectPassword: str) -> None:
         if self.isConnected():
             message = {'Operation': 'Open Project',
                        'Arguments': {'project_name': project_name, 'password': projectPassword}}
             self.transmitMessage(message)
 
-    def closeProject(self, project_name: str):
+    def closeProject(self, project_name: str) -> None:
         if self.isConnected():
             message = {'Operation': 'Close Project',
                        'Arguments': {'project_name': project_name}}
             self.transmitMessage(message)
 
-    def askProjectCanvasesList(self, project_name: str):
+    def askProjectCanvasesList(self, project_name: str) -> None:
         if self.isConnected():
             message = {'Operation': 'List Synced Canvases',
                        'Arguments': {'project_name': project_name}}
             self.transmitMessage(message)
 
-    def receiveProjectCanvasesList(self, canvases: list):
+    def receiveProjectCanvasesList(self, canvases: list) -> None:
         self.receive_project_canvases_list_signal.emit(canvases)
 
-    def askQuestion(self, project_name: str, question: str, reader_value: int, retriever_value: int, answer_count: int):
+    def askQuestion(self, project_name: str, question: str, reader_value: int, retriever_value: int,
+                    answer_count: int) -> None:
         if self.isConnected():
             message = {"Operation": "Ask Question",
                        "Arguments": {
@@ -378,21 +440,21 @@ class CommunicationsHandler(QtCore.QObject):
                            'answer_count': answer_count}}
             self.transmitMessage(message)
 
-    def receiveQuestionAnswer(self, answer: list):
+    def receiveQuestionAnswer(self, answer: list) -> None:
         self.receive_question_answer.emit(answer)
 
-    def receiveTextMessage(self, chat_message: str):
+    def receiveTextMessage(self, chat_message: str) -> None:
         self.receive_chat_message.emit(chat_message)
 
-    def sendTextMessage(self, project_name: str, chat_message: str):
+    def sendTextMessage(self, project_name: str, chat_message: str) -> None:
         if self.isConnected():
             message = {"Operation": "Chat",
                        "Arguments": {
                            "project_name": project_name,
-                           "chat_message": chat_message[:1024]}}
+                           "chat_message": chat_message[:1024]}}  # Cap messages to prevent spam.
             self.transmitMessage(message)
 
-    def syncDatabase(self, project_name: str, client_project_graph: nx.DiGraph):
+    def syncDatabase(self, project_name: str, client_project_graph: nx.DiGraph) -> None:
         message = {'Operation': 'Sync Database',
                    'Arguments': {
                        'project_name': project_name,
@@ -401,16 +463,16 @@ class CommunicationsHandler(QtCore.QObject):
                    }}
         self.transmitMessage(message)
 
-    def receiveSyncDatabase(self, database: str):
-        database_nodes, database_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(database)
+    def receiveSyncDatabase(self, database: str) -> None:
+        database_nodes, database_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraphFromString(database)
         self.receive_sync_database.emit(database_nodes, database_edges)
 
-    def askServerForFileList(self, project_name: str):
+    def askServerForFileList(self, project_name: str) -> None:
         message = {"Operation": "Get File List",
                    "Arguments": {'project_name': project_name}}
         self.transmitMessage(message)
 
-    def syncCanvasSend(self, project_name: str, canvas_name: str, canvas_graph: nx.DiGraph):
+    def syncCanvasSend(self, project_name: str, canvas_name: str, canvas_graph: nx.DiGraph) -> None:
         if self.isConnected():
             message = {'Operation': "Sync Canvas",
                        'Arguments': {
@@ -419,11 +481,11 @@ class CommunicationsHandler(QtCore.QObject):
                            "canvas_graph": str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(canvas_graph))}}
             self.transmitMessage(message)
 
-    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: str):
-        graph_nodes, graph_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(canvas_graph)
+    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: str) -> None:
+        graph_nodes, graph_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraphFromString(canvas_graph)
         self.receive_sync_canvas_signal.emit(canvas_name, graph_nodes, graph_edges)
 
-    def closeCanvas(self, project_name: str, canvas_name: str):
+    def closeCanvas(self, project_name: str, canvas_name: str) -> None:
         if self.isConnected():
             message = {'Operation': 'Close Canvas',
                        'Arguments': {'project_name': project_name,
@@ -437,18 +499,14 @@ class CommunicationsHandler(QtCore.QObject):
     #   to a particular canvas.
     # Being verbose is better than prematurely optimizing for a few kbps of
     #   network traffic.
-    def receiveDatabaseUpdateEvent(self, entity_json: dict, add: bool):
-        try:
+    def receiveDatabaseUpdateEvent(self, entity_json: dict, add: int) -> None:
+        with contextlib.suppress(KeyError):
             entity_json['Icon'] = QtCore.QByteArray(b64decode(entity_json['Icon']))
-        except KeyError:
-            pass
         self.receive_project_database_update.emit(entity_json, add)
 
-    def sendDatabaseUpdateEvent(self, project_name: str, entity_json: dict, add: bool):
-        try:
+    def sendDatabaseUpdateEvent(self, project_name: str, entity_json: dict, add: int) -> None:
+        with contextlib.suppress(KeyError):
             entity_json['Icon'] = entity_json['Icon'].toBase64().data()
-        except KeyError:
-            pass
         message = {"Operation": "Update Project Entities",
                    "Arguments": {
                        'project_name': project_name,
@@ -456,13 +514,14 @@ class CommunicationsHandler(QtCore.QObject):
                        "add": add}}
         self.transmitMessage(message)
 
-    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
+    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]) -> None:
         if isinstance(entity_or_link_uid, str):
             self.receive_project_canvas_update_node.emit(canvas_name, entity_or_link_uid)
         else:
             self.receive_project_canvas_update_link.emit(canvas_name, entity_or_link_uid)
 
-    def sendCanvasUpdateEvent(self, project_name: str, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
+    def sendCanvasUpdateEvent(self, project_name: str, canvas_name: str,
+                              entity_or_link_uid: Union[str, tuple]) -> None:
         message = {"Operation": "Update Canvas Entities",
                    "Arguments": {
                        'project_name': project_name,
@@ -470,10 +529,10 @@ class CommunicationsHandler(QtCore.QObject):
                        "entity_or_link_uid": entity_or_link_uid}}
         self.transmitMessage(message)
 
-    def receiveFileList(self, file_list: list):
+    def receiveFileList(self, file_list: list) -> None:
         self.receive_project_file_list.emit(file_list)
 
-    def sendFile(self, project_name: str, file_name: str, filePath: Path):
+    def sendFile(self, project_name: str, file_name: str, filePath: Path) -> None:
         """
         Starts a thread, calling sendFileHelper to send the file specified.
 
@@ -482,12 +541,12 @@ class CommunicationsHandler(QtCore.QObject):
         :param filePath:
         :return:
         """
-        sendHelperThread = threading.Thread(target=self.sendFileHelper, args=(project_name, file_name, filePath))
-        sendHelperThread.setDaemon(True)
+        sendHelperThread = threading.Thread(target=self.sendFileHelper, daemon=True,
+                                            args=(project_name, file_name, filePath))
         self.uploadingFiles[file_name] = sendHelperThread
         sendHelperThread.start()
 
-    def sendFileHelper(self, project_name: str, file_name: str, filePath: Path):
+    def sendFileHelper(self, project_name: str, file_name: str, filePath: Path) -> None:
         """
         Sends file in chunks to avoid loading the entire thing in memory.
 
@@ -498,31 +557,28 @@ class CommunicationsHandler(QtCore.QObject):
         """
         if not filePath.exists() or not filePath.is_file():
             return
-        fileHandler = open(filePath, 'rb')
-        print('Sending file:', filePath)
-
-        currThread = threading.currentThread()
-        while getattr(currThread, "continue_running", True):
-            filePart = fileHandler.read(512)
-            if not filePart:
-                messageJson = {"Operation": "File Upload Done",
+        with open(filePath, 'rb') as fileHandler:
+            currThread = threading.current_thread()
+            while getattr(currThread, "continue_running", True):
+                filePart = fileHandler.read(512)
+                if not filePart:
+                    messageJson = {"Operation": "File Upload Done",
+                                   "Arguments": {
+                                       'project_name': project_name,
+                                       'file_name': file_name
+                                   }}
+                    self.transmitMessage(messageJson)
+                    break
+                messageJson = {"Operation": "File Upload",
                                "Arguments": {
                                    'project_name': project_name,
-                                   'file_name': file_name
+                                   'file_name': file_name,
+                                   'file_contents': filePart
                                }}
                 self.transmitMessage(messageJson)
-                break
-            messageJson = {"Operation": "File Upload",
-                           "Arguments": {
-                               'project_name': project_name,
-                               'file_name': file_name,
-                               'file_contents': filePart
-                           }}
-            self.transmitMessage(messageJson)
-        fileHandler.close()
 
-    def sendFileAbort(self, project_name: str, file_name: str):
-        try:
+    def sendFileAbort(self, project_name: str, file_name: str) -> None:
+        with contextlib.suppress(KeyError):
             uploadToAbort = self.uploadingFiles.pop(file_name)
             uploadToAbort.continue_running = False
             messageJson = {"Operation": "File Upload Abort",
@@ -531,10 +587,8 @@ class CommunicationsHandler(QtCore.QObject):
                                'file_name': file_name
                            }}
             self.transmitMessage(messageJson)
-        except KeyError:
-            pass
 
-    def scanInbox(self):
+    def scanInbox(self) -> None:
         """
         This function checks if there is anything in the inbox, and if
         there is, calls the appropriate functions.
@@ -545,15 +599,14 @@ class CommunicationsHandler(QtCore.QObject):
                 message = self.inbox.get(timeout=0.2)
             except Empty:
                 with closeSoftwareLock:
-                    if not closeSoftware:
-                        time.sleep(0.2)
-                        continue
-                    else:
+                    if closeSoftware:
                         return
+                    time.sleep(0.2)
+                    continue
             if prevMesg == message:
                 # Same message, do not waste time handling.
                 continue
-            print('Message To handle:', message)  # TODO Make this logging.
+            self.mainWindow.MESSAGEHANDLER.debug(f'Message to handle: {str(message)}')
             operation = message['Operation']
             arguments = message['Arguments']
             if operation == 'Get Server Resolutions':
@@ -566,6 +619,8 @@ class CommunicationsHandler(QtCore.QObject):
                 self.receiveProjectCanvasesList(**arguments)
             elif operation == 'Resolution Result':
                 self.receiveResolutionResult(**arguments)
+            elif operation == 'Collector Results Signal':
+                self.receiveCollectorResult(**arguments)
             elif operation == "Chat":
                 self.receiveTextMessage(**arguments)
             elif operation == "Sync Database":
@@ -586,14 +641,16 @@ class CommunicationsHandler(QtCore.QObject):
                 self.receiveFileSummaryListener(**arguments)
             elif operation == "File Upload Abort":
                 self.receiveFileUploadAbort(**arguments)
-            elif operation == "Delete File":
-                pass
+            elif operation == "Get Server Collectors":
+                self.receiveCollectors(**arguments)
+            elif operation == "Start Collector":
+                self.receiveStartCollector(**arguments)
             else:
-                self.mainWindow.MESSAGEHANDLER.warning('Unhandled message: ' + str(message) +
-                                                       ' On Operation: ' + str(operation))
+                self.mainWindow.MESSAGEHANDLER.warning(f'Unhandled message: {str(message)} On Operation: '
+                                                       f'{str(operation)}')
             prevMesg = message
 
-    def handleStatusMessage(self, operation: str, message: str, status_code: int):
+    def handleStatusMessage(self, operation: str, message: str, status_code: int) -> None:
         """
         Operations that are completely server-side, or do not conform to the query - response model,
         send status messages to inform the client of what is going on.
@@ -608,90 +665,89 @@ class CommunicationsHandler(QtCore.QObject):
         :param status_code:
         :return:
         """
-        if status_code != 200:
-            self.status_message_signal.emit('Operation ' + operation + ' failed with status code ' +
-                                            str(status_code) + ': ' + message, True)
-        else:
-            if operation == 'Create Project':
-                # No need to do anything here. Creating a new project also opens it.
-                pass
-            elif operation == 'Open Project':
-                projectName = message.split(': ', 1)[1]
-                self.open_project_signal.emit(projectName)
-            # Show the user that the server is doing something.
-            elif operation == 'Opening Project':
-                self.status_message_signal.emit(message, True)
-            elif operation == 'Close Project':
-                self.close_project_signal.emit()
-            elif operation == 'Create Canvas':
-                # No need to do anything here. Creating a new canvas also opens it.
-                pass
-            elif operation == 'Open Canvas':
-                canvas_name = message.split(': ', 1)[1]
-                self.open_project_canvas_signal.emit(canvas_name)
-            elif operation == 'Close Canvas':
-                canvas_name = message.split(': ', 1)[1]
-                self.close_project_canvas_signal.emit(canvas_name)
-            elif operation == 'Connect To Server':
-                server_name = message.split(': ', 1)[1]
-                self.connected_to_server_listener.emit(server_name)
-            elif operation == 'File Upload':
-                file_name = message.split(': ', 1)[1]
-                self.file_upload_finished_signal.emit(file_name)
-            elif operation == 'File Download Done':
-                file_name = message.split(': ', 1)[1]
-                self.receiveFileDoneListener(file_name)
-            elif operation == 'Abort Resolution':
+        if status_code == 200:
+            if operation == 'Abort Resolution':
                 # Remove resolution from resolutions list.
                 resolution_uid = message.split(': ', 1)[1]
                 self.remove_server_resolution_from_running_signal.emit(resolution_uid)
-            elif operation == 'Delete File':
-                # Remove file from uploaded files list.
-                pass  # TODO
+            elif operation == 'Close Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.close_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Close Project':
+                self.close_project_signal.emit()
+            elif operation == 'Connect To Server':
+                server_name = message.split(': ', 1)[1]
+                self.connected_to_server_listener.emit(server_name)
+            elif operation in {'Create Project', 'Create Canvas', 'Stop Collector'}:
+                # No need to do anything for these.
+                pass
+            elif operation == 'Delete Project':
+                # Remove project from server projects list.
+                project_name = message.split(': ', 1)[1]
+                self.delete_server_project_signal.emit(project_name)
+            elif operation == 'File Download Done':
+                file_name = message.split(': ', 1)[1]
+                self.receiveFileDoneListener(file_name)
             elif operation == 'File Upload Abort':
                 file_name = message.split(': ', 1)[1]
                 # Remove file from uploading files list.
                 self.file_upload_abort_signal.emit(file_name)
+            elif operation == 'File Upload':
+                file_name = message.split(': ', 1)[1]
+                self.file_upload_finished_signal.emit(file_name)
+            elif operation == 'Open Canvas':
+                canvas_name = message.split(': ', 1)[1]
+                self.open_project_canvas_signal.emit(canvas_name)
+            elif operation == 'Open Project':
+                projectName = message.split(': ', 1)[1]
+                self.open_project_signal.emit(projectName)
+            elif operation == 'Opening Project':
+                self.status_message_signal.emit(message, True)
             else:
-                self.mainWindow.MESSAGEHANDLER.warning('Unhandled status message: ' + message +
-                                                       ' Code: ' + str(status_code) +
-                                                       ' On Operation: ' + str(operation))
+                self.mainWindow.MESSAGEHANDLER.warning(f'Unhandled status message: {message} Code: {status_code} '
+                                                       f'On Operation: {operation}')
 
-    def receiveFile(self, project_name: str, file_name: str):
+        elif status_code == 404 and message == 'No project with the specified name exists!':
+            self.close_project_signal.emit()
+        else:
+            self.status_message_signal.emit(f'Operation {operation} failed with status code {status_code}: {message}',
+                                            True)
+
+    def receiveFile(self, project_name: str, file_name: str, saveDir: Path) -> None:
         # Do not download files already being downloaded.
         if self.downloadingFiles.get(file_name) is None:
-            saveDir = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
             fileHandler = open(saveDir, "wb")
             self.downloadingFiles[file_name] = fileHandler
 
-            message = {'Operation': 'Download File',
+            message = {'Operation': 'File Download',
                        'Arguments': {
                            'project_name': project_name,
                            'file_name': file_name
                        }}
             self.transmitMessage(message)
 
-    def receiveFileListener(self, file_name: str, file_contents: bytes):
+    def receiveFileListener(self, file_name: str, file_contents: bytes) -> None:
         try:
             fileHandler = self.downloadingFiles.get(file_name)
             fileHandler.write(file_contents)
         except Exception:
-            # In case the file is deleted in the middle of writing, or anything else going wrong.
-            self.mainWindow.MESSAGEHANDLER.warning('Received data for file: ' + file_name +
-                                                   ' but no valid file handler exists for this file.')
+            # In case something goes wrong in the middle of writing.
+            self.mainWindow.MESSAGEHANDLER.warning(f'Received data for file: {file_name} but no valid file handler '
+                                                   f'exists for this file.')
 
-    def receiveFileDoneListener(self, file_name: str):
+    def receiveFileDoneListener(self, file_name: str) -> None:
         fileHandler = self.downloadingFiles.pop(file_name)
         if fileHandler is None:
-            self.mainWindow.MESSAGEHANDLER.warning('Received file: ' + file_name +
-                                                   ' but no file handler exists for this file.')
+            self.mainWindow.MESSAGEHANDLER.warning(f'Received file: {file_name} but no file handler exists for this '
+                                                   f'file.')
+
             return
 
         fileHandler.close()
-        self.status_message_signal.emit('Finished downloading file from server: ' + file_name, False)
+        self.status_message_signal.emit(f'Finished downloading file from server: {file_name}', True)
 
-    def receiveFileAbort(self, project_name: str, file_name: str):
-        messageJson = {"Operation": "Download File Abort",
+    def receiveFileAbort(self, project_name: str, file_name: str) -> None:
+        messageJson = {"Operation": "File Download Abort",
                        "Arguments": {
                            'project_name': project_name,
                            'file_name': file_name
@@ -704,24 +760,23 @@ class CommunicationsHandler(QtCore.QObject):
         abortedPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")) / file_name
         abortedPath.unlink(missing_ok=True)
 
-    def deleteFile(self, project_name: str, file_name: str):
-        message = {"Operation": "Delete File",
+    def deleteProject(self, project_name: str) -> None:
+        message = {"Operation": "Delete Project",
                    "Arguments": {
-                       'project_name': project_name,
-                       'file_name': file_name}}
+                       'project_name': project_name}}
         self.transmitMessage(message)
 
-    def askServerForFileSummary(self, project_name: str, document_name: str):
+    def askServerForFileSummary(self, project_name: str, document_name: str) -> None:
         message = {"Operation": "Get File Summary",
                    "Arguments": {
                        'project_name': project_name,
                        'document_name': document_name}}
         self.transmitMessage(message)
 
-    def receiveFileSummaryListener(self, document_name: str, summary: str):
+    def receiveFileSummaryListener(self, document_name: str, summary: str) -> None:
         self.receive_document_summary_signal.emit(document_name, summary)
 
-    def receiveFileUploadAbort(self, file_name: str):
+    def receiveFileUploadAbort(self, file_name: str) -> None:
         """
         If we are told by the server to stop uploading a file,
         we should do so (i.e. because no space left on server),
@@ -730,22 +785,20 @@ class CommunicationsHandler(QtCore.QObject):
         :param file_name:
         :return:
         """
-        try:
+        with contextlib.suppress(KeyError):
             uploadToAbort = self.uploadingFiles.pop(file_name)
             uploadToAbort.continue_running = False
             self.file_upload_abort_signal.emit(file_name)
-        except KeyError:
-            pass
 
-    def sendFileAbortAll(self, project_name: str):
+    def sendFileAbortAll(self, project_name: str) -> None:
         """
         Abort the sending of all files currently being transmitted.
 
         :param project_name:
         :return:
         """
-        for file_name in self.uploadingFiles:
-            try:
+        for file_name in dict(self.uploadingFiles):
+            with contextlib.suppress(KeyError):
                 uploadToAbort = self.uploadingFiles.pop(file_name)
                 uploadToAbort.continue_running = False
                 messageJson = {"Operation": "File Upload Abort",
@@ -754,18 +807,16 @@ class CommunicationsHandler(QtCore.QObject):
                                    'file_name': file_name
                                }}
                 self.transmitMessage(messageJson)
-            except KeyError:
-                pass
 
-    def receiveFileAbortAll(self, project_name: str):
+    def receiveFileAbortAll(self, project_name: str) -> None:
         """
         Abort the downloading of all files currently being transmitted.
 
         :param project_name:
         :return:
         """
-        for file_name in self.downloadingFiles:
-            messageJson = {"Operation": "Download File Abort",
+        for file_name in dict(self.downloadingFiles):
+            messageJson = {"Operation": "File Download Abort",
                            "Arguments": {
                                'project_name': project_name,
                                'file_name': file_name

Core/GlobalVariables.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import random
+
+non_string_fields = ('Icon', 'Child UIDs')
+hidden_fields = ('uid', 'Date Last Edited', 'Child UIDs', 'Canvas Banner', 'Entity Type')
+hidden_fields_dockbars = ('uid', 'Child UIDs', 'Canvas Banner', 'Icon')
+meta_fields = ('Child UIDs',)
+avoid_parsing_fields = ('uid', 'Date Last Edited', 'Child UIDs', 'Icon', 'Canvas Banner')
+
+# Closer to the top means more recent.
+user_agents = {'Chrome': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                                      '(KHTML, like Gecko) Chrome/101.0.4951.15 Safari/537.36',),
+                          'Linux': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                                    'Chrome/101.0.4951.15 Safari/537.36',)
+                          },
+               'Firefox': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; rv:100.0) Gecko/20100101 Firefox/98.0',),
+                           'Linux': ('Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',)
+                           },
+               'Webkit': {'Windows': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 '
+                                      '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',),
+                          'Linux': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 '
+                                    '(KHTML, like Gecko) Version/15.4 Safari/605.1.15',)
+                          }
+               }
+
+
+def getLatestUserAgent(browser: str = None, platform: str = None) -> str:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][0]
+
+
+def getUserAgents(browser: str = None, platform: str = None, amount: int = 1) -> tuple:
+    if browser is None or browser not in user_agents:
+        browser = random.choice(list(user_agents.keys()))
+    browserPlatforms = user_agents[browser]
+    if platform is None or platform not in browserPlatforms:
+        platform = random.choice(list(browserPlatforms.keys()))
+    return user_agents[browser][platform][:amount]

Core/Interface/DockBarOne.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
 from json import dumps
 from Core.Interface.Entity import BaseNode
-from Core.Interface import Stylesheets
 from PySide6 import QtWidgets, QtCore, QtGui
 
+from Core.ResourceHandler import resizePictureFromBuffer
+
 
 class DockBarOne(QtWidgets.QDockWidget):
 
-    def initialiseLayout(self):
+    def initialiseLayout(self) -> None:
         childWidget = QtWidgets.QTabWidget()
         nodeTabChildWidget = QtWidgets.QWidget()
         docsTabChildWidget = QtWidgets.QWidget()
@@ -31,16 +32,16 @@ class DockBarOne(QtWidgets.QDockWidget):
                  resolutionManager,
                  resourceHandler,
                  entityDatabase,
-                 title="DockBar One"):
+                 title="DockBar One") -> None:
         super(DockBarOne, self).__init__(parent=mainWindow)
         self.resolutionManager = resolutionManager
         self.resourceHandler = resourceHandler
         self.lentDB = entityDatabase
-        self.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
-                             QtCore.Qt.RightDockWidgetArea)
-        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
-                         QtWidgets.QDockWidget.DockWidgetFloatable |
-                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.LeftDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
         self.setWindowTitle(title)
         self.setObjectName(title)
 
@@ -57,13 +58,13 @@ class DockBarOne(QtWidgets.QDockWidget):
 
 class ToggleWorldDocButton(QtWidgets.QPushButton):
 
-    def __init__(self, parent=None, mainWindow=None):
+    def __init__(self, parent=None, mainWindow=None) -> None:
         super(ToggleWorldDocButton, self).__init__(parent=parent)
         self.mainWindow = mainWindow
         self.setText("Toggle Detailed View")
         self.clicked.connect(self.buttonPressed)
 
-    def buttonPressed(self):
+    def buttonPressed(self) -> None:
         if self.mainWindow is not None:
             self.mainWindow.toggleWorldDoc()
 
@@ -79,7 +80,7 @@ class EntityList(QtWidgets.QTreeWidget):
         self.setHeaderLabels(['Entity List'])
         self.setAlternatingRowColors(False)
         self.setMinimumWidth(200)
-        self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
+        self.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection)
         self.menu = QtWidgets.QMenu()
 
         actionDelete = QtGui.QAction('Delete Selected Items',
@@ -87,13 +88,18 @@ class EntityList(QtWidgets.QTreeWidget):
                                      statusTip="Delete the selected entities from the database.",
                                      triggered=self.deleteSelectedItems)
         self.menu.addAction(actionDelete)
-        self.menu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
+
+        actionAddToCurrentCanvas = QtGui.QAction('Add Selected Items to Current Canvas',
+                                                 self.menu,
+                                                 statusTip="Add the selected entities to the current canvas.",
+                                                 triggered=self.addItemsToCurrentCanvas)
+        self.menu.addAction(actionAddToCurrentCanvas)
 
         self.entityCategories: dict = {}
         self.entityTypes: dict = {}
         self.loadEntities()
 
-    def loadEntities(self):
+    def loadEntities(self) -> None:
         self.clear()
         self.entityCategories = {}
         self.entityTypes = {}
@@ -112,7 +118,8 @@ class EntityList(QtWidgets.QTreeWidget):
             topItem = self.entityTypes[entity['Entity Type']]
             primaryAttr = entity[list(entity)[1]]
             pixmapIcon = QtGui.QPixmap()
-            pixmapIcon.loadFromData(entity['Icon'])
+            resizedIcon = resizePictureFromBuffer(entity['Icon'], (40, 40))
+            pixmapIcon.loadFromData(resizedIcon)
             EntityWidget(topItem,
                          entity['uid'],
                          QtGui.QIcon(pixmapIcon),
@@ -129,7 +136,7 @@ class EntityList(QtWidgets.QTreeWidget):
             if childrenHidden == self.entityCategories[category].childCount():
                 self.entityCategories[category].setHidden(True)
 
-    def addEntity(self, entityJson):
+    def addEntity(self, entityJson) -> None:
         primaryAttr = entityJson[list(entityJson)[1]]
         entityTypeItem = self.entityTypes[entityJson['Entity Type']]
         for entityNo in range(entityTypeItem.childCount()):
@@ -138,7 +145,8 @@ class EntityList(QtWidgets.QTreeWidget):
                 child.setText(0, primaryAttr)
                 return
         pixmapIcon = QtGui.QPixmap()
-        pixmapIcon.loadFromData(entityJson['Icon'])
+        resizedIcon = resizePictureFromBuffer(entityJson['Icon'], (40, 40))
+        pixmapIcon.loadFromData(resizedIcon)
         EntityWidget(entityTypeItem,
                      entityJson['uid'],
                      QtGui.QIcon(pixmapIcon),
@@ -148,7 +156,7 @@ class EntityList(QtWidgets.QTreeWidget):
             entityTypeItem.setHidden(False)
             entityTypeItem.parent().setHidden(False)
 
-    def removeEntity(self, entityJson):
+    def removeEntity(self, entityJson) -> None:
         entityType = self.entityTypes[entityJson['Entity Type']]
         for entityNo in range(entityType.childCount()):
             child = entityType.child(entityNo)
@@ -166,35 +174,36 @@ class EntityList(QtWidgets.QTreeWidget):
                             break
                 break
 
-    def mouseMoveEvent(self, event):
+    def mouseMoveEvent(self, event) -> None:
         """
         Handle dragging of entities onto canvas.
         """
-        # No, I have no idea why this is the case: v
-        if event.button() == QtGui.Qt.MouseButton.NoButton:
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
             itemDragged = self.itemAt(event.pos())
-
-            if itemDragged is None or \
-                    itemDragged.text(0) in self.entityTypes.keys() or \
-                    itemDragged.text(0) in self.entityCategories.keys():
-                return
-
-            self.setCurrentItem(itemDragged)
-            drag = QtGui.QDrag(self)
-            mimeData = QtCore.QMimeData()
-
-            mimeData.setText(dumps({'uid': itemDragged.uid}))
-            drag.setMimeData(mimeData)
-
-            # All Entities should have icons, but you never know.
-            pixmap = None
-            if itemDragged.icon(0) is not None:
-                pixmap = itemDragged.icon(0).pixmap(40, 40)
-                drag.setPixmap(pixmap)
-            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
-            drag.exec_()
         else:
-            super().mouseMoveEvent(event)
+            return
+
+        # Categories & entity names don't have uids.
+        try:
+            if itemDragged.uid is None:
+                return
+        except Exception:
+            return
+
+        self.setCurrentItem(itemDragged)
+        drag = QtGui.QDrag(self)
+        mimeData = QtCore.QMimeData()
+
+        mimeData.setText(dumps({'uid': itemDragged.uid}))
+        drag.setMimeData(mimeData)
+
+        # All Entities should have icons, but you never know.
+        pixmap = None
+        if itemDragged.icon(0) is not None:
+            pixmap = itemDragged.icon(0).pixmap(40, 40)
+            drag.setPixmap(pixmap)
+        drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() // 2, pixmap.rect().height() // 2))
+        drag.exec_()
 
     def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
         super(EntityList, self).mousePressEvent(event)
@@ -204,15 +213,28 @@ class EntityList(QtWidgets.QTreeWidget):
             if isinstance(itemDragged, EntityWidget):
                 self.menu.exec(QtGui.QCursor.pos())
 
-    def deleteSelectedItems(self):
+    def deleteSelectedItems(self) -> None:
         itemsToDel = [item.uid for item in self.selectedItems() if isinstance(item, EntityWidget)]
         for itemUID in itemsToDel:
             self.mainWindow.deleteSpecificEntity(itemUID)
 
+    def addItemsToCurrentCanvas(self) -> None:
+        currentScene = self.mainWindow.centralWidget().tabbedPane.getCurrentScene()
+        itemsToAdd = [item.uid for item in self.selectedItems() if isinstance(item, EntityWidget) and
+                      item.uid not in currentScene.sceneGraph.nodes]
+        for itemUID in itemsToAdd:
+            if itemUID.endswith('@'):
+                newGroupEntity = self.mainWindow.copyGroupEntity(itemUID, currentScene)
+                if newGroupEntity is not None:
+                    currentScene.addNodeProgrammatic(newGroupEntity['uid'], newGroupEntity['Child UIDs'])
+            else:
+                currentScene.addNodeProgrammatic(itemUID)
+        currentScene.rearrangeGraph()
+
 
 class EntityWidget(QtWidgets.QTreeWidgetItem):
 
-    def __init__(self, parent, uid, icon=None, text=""):
+    def __init__(self, parent, uid=None, icon=None, text="") -> None:
         super(EntityWidget, self).__init__(parent, [text])
         self.uid = uid
         if icon is not None:
@@ -221,37 +243,30 @@ class EntityWidget(QtWidgets.QTreeWidgetItem):
 
 class DocList(QtWidgets.QTreeWidget):
 
-    def __init__(self, resourceHandler, parent=None):
+    def __init__(self, resourceHandler, parent=None) -> None:
         super(DocList, self).__init__(parent=parent)
+
         self.resourceHandler = resourceHandler
         self.setAlternatingRowColors(False)
         self.setHeaderLabels(['Files Loaded'])
         self.uploadingFileWidgets = []
         self.uploadedFileWidgets = []
-        self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
-        self.menu = QtWidgets.QMenu()
+        self.setSelectionMode(QtWidgets.QAbstractItemView.SelectionMode.ExtendedSelection)
 
-        actionDelete = QtGui.QAction('Delete Selected Files',
-                                     self.menu,
-                                     statusTip="Delete the selected files from the server.",
-                                     triggered=self.deleteSelectedFiles)
-        self.menu.addAction(actionDelete)
-        self.menu.setStyleSheet(Stylesheets.MENUS_STYLESHEET_2)
-
-    def addUploadingFileToList(self, fileName: str):
+    def addUploadingFileToList(self, fileName: str) -> None:
         newWidget = DocWidget(self,
                               QtGui.QIcon(self.resourceHandler.getIcon("uploading")), fileName)
         self.uploadingFileWidgets.append(newWidget)
 
-    def finishUploadingFile(self, fileName: str):
+    def finishUploadingFile(self, fileName: str) -> None:
         # Remove uploading file. Server will send updated file list.
-        for doc in self.uploadingFileWidgets:
+        for doc in list(self.uploadingFileWidgets):
             if doc.getFileName() == fileName:
                 self.takeTopLevelItem(self.indexOfTopLevelItem(doc))
                 self.uploadingFileWidgets.remove(doc)
                 break
 
-    def updateFileListFromServer(self, fileList):
+    def updateFileListFromServer(self, fileList) -> None:
         # If disconnected from server, or project has an empty file list, clear the widget.
         if not fileList:
             self.clear()
@@ -265,26 +280,15 @@ class DocList(QtWidgets.QTreeWidget):
             newWidget = DocWidget(self, None, fileName)
             self.uploadedFileWidgets.append(newWidget)
 
-    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
-        super(DocList, self).mousePressEvent(event)
-        # TODO - be able to drag downloaded files onto canvas, and create entities when that happens.
-
-    def deleteSelectedFiles(self):
-        filesToDel = [file.getFileName() for file in self.selectedItems() if isinstance(file, DocWidget)]
-        for fileName in filesToDel:
-            # TODO
-            pass
-            # self.mainWindow.deleteSpecificEntity(itemUID)
-
 
 class DocWidget(QtWidgets.QTreeWidgetItem):
 
-    def __init__(self, parent, icon=None, text=""):
+    def __init__(self, parent, icon=None, text="") -> None:
         super(DocWidget, self).__init__(parent, [text])
         if icon is not None:
             self.setIcon(0, icon)
 
-    def getFileName(self):
+    def getFileName(self) -> str:
         return self.text(0)
 
 
@@ -294,9 +298,10 @@ class ResolutionList(QtWidgets.QTreeWidget):
                  resolutionManager,
                  entityDatabase,
                  mainWindow,
-                 parent=None):
+                 parent=None) -> None:
 
         super(ResolutionList, self).__init__(parent=parent)
+
         self.resolutionManager = resolutionManager
         self.lentDB = entityDatabase
         self.mainWindow = mainWindow
@@ -304,10 +309,12 @@ class ResolutionList(QtWidgets.QTreeWidget):
         self.setHeaderLabels(['Resolutions'])
         self.setAlternatingRowColors(False)
         self.setMinimumWidth(200)
+        self.setSortingEnabled(True)
+        self.sortByColumn(0, QtCore.Qt.SortOrder.AscendingOrder)
 
         self.loadAllResolutions()
 
-    def loadAllResolutions(self):
+    def loadAllResolutions(self) -> None:
         self.clear()
         for category in self.resolutionManager.getResolutionCategories():
             resTreeItem = QtWidgets.QTreeWidgetItem(self, [category])
@@ -315,7 +322,7 @@ class ResolutionList(QtWidgets.QTreeWidget):
             for res in resolutions:
                 ResolutionWidget(resTreeItem, text=res)
 
-    def loadResolutionsForSelected(self, selected):
+    def loadResolutionsForSelected(self, selected) -> None:
         self.clear()
         if len(selected) == 0:
             self.loadAllResolutions()
@@ -334,7 +341,7 @@ class ResolutionList(QtWidgets.QTreeWidget):
                 for resolution in resolutions:
                     ResolutionWidget(resTreeItem, text=resolution)
 
-    def mouseDoubleClickEvent(self, event):
+    def mouseDoubleClickEvent(self, event) -> None:
         super(ResolutionList, self).mouseDoubleClickEvent(event)
         resItem = self.itemAt(event.pos())
         if resItem is None or not isinstance(resItem, ResolutionWidget):
@@ -345,12 +352,12 @@ class ResolutionList(QtWidgets.QTreeWidget):
         if resolution not in self.resolutionManager.getResolutionsInCategory(category):
             return
 
-        self.mainWindow.runResolution(category + '/' + resolution)
+        self.mainWindow.runResolution(f'{category}/{resolution}')
 
 
 class ResolutionWidget(QtWidgets.QTreeWidgetItem):
 
-    def __init__(self, parent, icon=None, text=""):
+    def __init__(self, parent, icon=None, text="") -> None:
         super(ResolutionWidget, self).__init__(parent, [text])
         if icon is not None:
             self.setIcon(0, icon)
@@ -358,17 +365,20 @@ class ResolutionWidget(QtWidgets.QTreeWidgetItem):
 
 class NodeList(QtWidgets.QTreeWidget):
 
-    def __init__(self, resourceHandler, parent=None):
+    def __init__(self, resourceHandler, parent=None) -> None:
         super(NodeList, self).__init__(parent=parent)
+
         self.resourceHandler = resourceHandler
         self.setDragEnabled(True)
         self.setHeaderLabels(['Entities'])
         self.setAlternatingRowColors(False)
+        self.setSortingEnabled(True)
+        self.sortByColumn(0, QtCore.Qt.SortOrder.AscendingOrder)
         self.allEntities = []
 
         self.loadEntities()
 
-    def loadEntities(self):
+    def loadEntities(self) -> None:
         self.clear()
         self.allEntities = []
         for category in self.resourceHandler.getEntityCategories():
@@ -383,12 +393,11 @@ class NodeList(QtWidgets.QTreeWidget):
 
         self.allEntities = self.resourceHandler.getAllEntities()
 
-    def mouseMoveEvent(self, event):
+    def mouseMoveEvent(self, event) -> None:
         """
         Handle dragging of entities onto canvas.
         """
-        # No, I have no idea why this is the case: v
-        if event.button() == QtGui.Qt.MouseButton.NoButton:
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
             itemDragged = self.itemAt(event.pos())
             if itemDragged is None or \
                     itemDragged.text(0) not in self.allEntities:
@@ -406,16 +415,13 @@ class NodeList(QtWidgets.QTreeWidget):
             if itemDragged.icon(0) is not None:
                 pixmap = itemDragged.icon(0).pixmap(40, 40)
                 drag.setPixmap(pixmap)
-            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
+            drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() // 2, pixmap.rect().height() // 2))
             drag.exec_()
-        else:
-            # This should never happen.
-            super(NodeList, self).mousePressEvent(event)
 
 
 class NodeWidget(QtWidgets.QTreeWidgetItem):
 
-    def __init__(self, parent, icon=None, name="", entityJsonText=""):
+    def __init__(self, parent, icon=None, name="", entityJsonText="") -> None:
         super(NodeWidget, self).__init__(parent, [name])
         if icon is not None:
             self.setIcon(0, icon)

Core/Interface/DockBarThree.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
 
+import contextlib
 from PySide6 import QtWidgets, QtCore, QtCharts, QtGui
-from Core.Interface import Stylesheets
 from datetime import datetime
 from getpass import getuser
-from time import sleep
 import networkx as nx
 import queue
 
@@ -15,7 +14,6 @@ class DockBarThree(QtWidgets.QDockWidget):
     """
 
     def initialiseLayout(self):
-        # self.setStyleSheet(Stylesheets.MAIN_WINDOW_STYLESHEET)
         childWidget = QtWidgets.QWidget()
         childWidget.setLayout(QtWidgets.QVBoxLayout())
         childWidget.setContentsMargins(0, 0, 0, 0)
@@ -30,20 +28,18 @@ class DockBarThree(QtWidgets.QDockWidget):
         self.tabPane.addTab(self.logViewer, 'Program Log')
         self.tabPane.addTab(self.timeWidget, 'Timeline')
         childWidget2.layout().addWidget(self.chatBox)
-        self.serverStatus.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
         childWidget.layout().addWidget(self.serverStatus)
 
     def __init__(self, mainWindow, title="Dockbar Three"):
         super(DockBarThree, self).__init__(parent=mainWindow)
-        self.setAllowedAreas(QtCore.Qt.TopDockWidgetArea |
-                             QtCore.Qt.BottomDockWidgetArea)
-        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
-                         QtWidgets.QDockWidget.DockWidgetFloatable |
-                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.TopDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.BottomDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
         self.setWindowTitle(title)
         self.setObjectName(title)
-        self.setMaximumHeight(275)
-        self.setMinimumHeight(275)
+        self.setMinimumHeight(300)
 
         self.tabPane = QtWidgets.QTabWidget()
 
@@ -76,11 +72,11 @@ class TimeWidget(QtWidgets.QWidget):
 
         self.timelineChart = QtCharts.QChart()
         self.timelineChart.setTitle("Timeline")
-        self.timelineChart.setTheme(QtCharts.QChart.ChartThemeBlueCerulean)
+        self.timelineChart.setTheme(QtCharts.QChart.ChartTheme.ChartThemeBlueCerulean)
         self.timelineChart.setMargins(QtCore.QMargins(0, 0, 0, 0))
         self.chartView = QtCharts.QChartView(self.timelineChart)
-        self.chartView.setRubberBand(QtCharts.QChartView.NoRubberBand)
-        self.timelineChart.setAnimationOptions(QtCharts.QChart.AllAnimations)
+        self.chartView.setRubberBand(QtCharts.QChartView.RubberBand.NoRubberBand)
+        self.timelineChart.setAnimationOptions(QtCharts.QChart.AnimationOption.AllAnimations)
         self.timelineChart.setAnimationDuration(250)
         self.timelineChart.legend().hide()
 
@@ -95,7 +91,10 @@ class TimeWidget(QtWidgets.QWidget):
         # Ref: https://qtcentre.org/threads/10975-Help-Export-QGraphicsView-to-Image-File
         # Rendering best optimized to rgb32 and argb32_premultiplied.
         # Ref: https://doc.qt.io/qtforpython/PySide6/QtGui/QImage.html?highlight=qimage#image-formats
-        picture = QtGui.QImage(self.chartView.size(), QtGui.QImage.Format_ARGB32_Premultiplied)
+        picture = QtGui.QImage(self.chartView.size(), QtGui.QImage.Format.Format_ARGB32_Premultiplied)
+        # Pictures are initialised with junk data - need to clear it out before painting
+        #   to avoid visual artifacts.
+        picture.fill(QtGui.QColor(0, 0, 0, 0))
         picturePainter = QtGui.QPainter(picture)
         if not transparentBackground:
             picture.fill(QtGui.QColor(61, 61, 61))
@@ -126,16 +125,12 @@ class TimeWidget(QtWidgets.QWidget):
 
         if nodeMinute not in self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour]:
             # Sanity check. Should not be able to remove nodes that do not exist, but you never know.
-            if added:
-                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 1
-            else:
-                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 0
-        else:
-            if added:
-                self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] += 1
-            else:
-                if self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] > 0:
-                    self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] -= 1
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] = 1 if added else 0
+
+        elif added:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] += 1
+        elif self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] > 0:
+            self.timeDetails[nodeYear][nodeMonth][nodeDay][nodeHour][nodeMinute] -= 1
 
         if updateGraph:
             self.drawChart([])
@@ -174,15 +169,11 @@ class TimeWidget(QtWidgets.QWidget):
         if minute is not None:
             barsDict = {minute: self.timeDetails[year][month][day][hour][minute]}
             self.currentTimeStep = [year, month, day, hour, minute]
-            self.drawChartHelper(barsDict, self.currentTimeStep)
-
         elif hour is not None:
-            barsDict = {}
-            for minute in self.timeDetails[year][month][day][hour]:
-                barsDict[minute] = self.timeDetails[year][month][day][hour][minute]
-            self.currentTimeStep = [year, month, day, hour]
-            self.drawChartHelper(barsDict, self.currentTimeStep)
+            barsDict = {minute: self.timeDetails[year][month][day][hour][minute]
+                        for minute in self.timeDetails[year][month][day][hour]}
 
+            self.currentTimeStep = [year, month, day, hour]
         elif day is not None:
             barsDict = {}
             for hour in self.timeDetails[year][month][day]:
@@ -190,8 +181,6 @@ class TimeWidget(QtWidgets.QWidget):
                 for minute in self.timeDetails[year][month][day][hour]:
                     barsDict[hour] += self.timeDetails[year][month][day][hour][minute]
             self.currentTimeStep = [year, month, day]
-            self.drawChartHelper(barsDict, self.currentTimeStep)
-
         elif month is not None:
             barsDict = {}
             for day in self.timeDetails[year][month]:
@@ -200,8 +189,6 @@ class TimeWidget(QtWidgets.QWidget):
                     for minute in self.timeDetails[year][month][day][hour]:
                         barsDict[day] += self.timeDetails[year][month][day][hour][minute]
             self.currentTimeStep = [year, month]
-            self.drawChartHelper(barsDict, self.currentTimeStep)
-
         elif year is not None:
             barsDict = {}
             for month in self.timeDetails[year]:
@@ -211,8 +198,6 @@ class TimeWidget(QtWidgets.QWidget):
                         for minute in self.timeDetails[year][month][day][hour]:
                             barsDict[month] += self.timeDetails[year][month][day][hour][minute]
             self.currentTimeStep = [year]
-            self.drawChartHelper(barsDict, self.currentTimeStep)
-
         else:
             barsDict = {}
             for year in self.timeDetails:
@@ -223,7 +208,8 @@ class TimeWidget(QtWidgets.QWidget):
                             for minute in self.timeDetails[year][month][day][hour]:
                                 barsDict[year] += self.timeDetails[year][month][day][hour][minute]
             self.currentTimeStep = []
-            self.drawChartHelper(barsDict, self.currentTimeStep)
+
+        self.drawChartHelper(barsDict, self.currentTimeStep)
 
     def drawChartHelper(self, barsDict: dict, timestep: list):
         timelineSeries = QtCharts.QBarSeries(self.timelineChart)
@@ -231,21 +217,20 @@ class TimeWidget(QtWidgets.QWidget):
         xAxisValues = []
 
         barSet = TimelineBarSet('Entities', self, timestep, list(barsDict))
-        barSet.setColor(QtGui.Qt.darkCyan)
+        barSet.setColor(QtGui.Qt.GlobalColor.darkCyan)
         for bar in barsDict:
             barSet.append(barsDict[bar])
             timelineSeries.append(barSet)
             if barsDict[bar] > maxEntityNum:
                 maxEntityNum = barsDict[bar]
             value = ""
-            for step in range(len(timestep)):
+            for step, stepValue in enumerate(timestep):
                 if step <= 2:
-                    value += str(timestep[step]) + '/'
+                    value += f'{str(stepValue)}/'
                     if step == 2:
-                        value = value[:-1]
-                        value += " "
+                        value = f"{value[:-1]} "
                 else:
-                    value += str(timestep[step]) + ':'
+                    value += f'{str(stepValue)}:'
             value += str(bar)
             xAxisValues.append(value)
         self.timelineChart.removeAllSeries()
@@ -255,7 +240,6 @@ class TimeWidget(QtWidgets.QWidget):
         self.timelineChart.removeAxis(self.timelineChart.axisY(timelineSeries))
 
         yAxis = QtCharts.QValueAxis()
-        yAxis.applyNiceNumbers()
         yAxis.setTickCount(min(maxEntityNum + 1, 4))
 
         xAxis = QtCharts.QBarCategoryAxis()
@@ -274,12 +258,10 @@ class TimeWidget(QtWidgets.QWidget):
             self.timescaleSelector.adjustLabelsToHour()
         elif timesteps == 4:
             self.timescaleSelector.adjustLabelsToMinute()
-        elif timesteps == 5:
-            self.timescaleSelector.adjustLabelsToSecond()
         else:
-            # Just in case.
             self.timescaleSelector.adjustLabelsToSecond()
 
+        yAxis.applyNiceNumbers()
         self.mainWindow.timelineSelectMatchingEntities(timestep)
 
 
@@ -302,35 +284,33 @@ class TimelineTimescaleSelector(QtWidgets.QLabel):
 
     def __init__(self, timeWidget: TimeWidget):
         super(TimelineTimescaleSelector, self).__init__(parent=timeWidget)
-        self.setStyleSheet("""border: 2px solid rgb(44, 49, 58);""")
 
         self.timeWidget = timeWidget
         self.setMinimumWidth(150)
-        self.setMaximumHeight(150)
 
         self.setLayout(QtWidgets.QFormLayout())
-        self.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
 
         self.yearButton = QtWidgets.QPushButton(' Year: ')
         self.yearButton.clicked.connect(self.yearButtonPressed)
         self.yearText = QtWidgets.QLabel('-')
-        self.yearText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.yearText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
         self.monthButton = QtWidgets.QPushButton(' Month: ')
         self.monthButton.clicked.connect(self.monthButtonPressed)
         self.monthText = QtWidgets.QLabel('X')
-        self.monthText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.monthText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
         self.dayButton = QtWidgets.QPushButton(' Day: ')
         self.dayButton.clicked.connect(self.dayButtonPressed)
         self.dayText = QtWidgets.QLabel('X')
-        self.dayText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.dayText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
         self.hourButton = QtWidgets.QPushButton(' Hour: ')
         self.hourButton.clicked.connect(self.hourButtonPressed)
         self.hourText = QtWidgets.QLabel('X')
-        self.hourText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.hourText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
         self.minuteButton = QtWidgets.QPushButton(' Minute: ')
         self.minuteButton.clicked.connect(self.minuteButtonPressed)
         self.minuteText = QtWidgets.QLabel('X')
-        self.minuteText.setFrameStyle(QtWidgets.QFrame.Sunken)
+        self.minuteText.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken)
 
         self.layout().addRow(self.yearButton, self.yearText)
         self.layout().addRow(self.monthButton, self.monthText)
@@ -475,8 +455,8 @@ class ServerStatusBox(QtWidgets.QLabel):
 
     def __init__(self, parent):
         super(ServerStatusBox, self).__init__(parent=parent)
-        self.setAlignment(QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
-        self.setFrameStyle(QtWidgets.QFrame.Sunken | QtWidgets.QFrame.StyledPanel)
+        self.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.setFrameStyle(QtWidgets.QFrame.Shadow.Sunken | QtWidgets.QFrame.Shape.StyledPanel)
         self.setText("Not connected to a server")
 
     def updateStatus(self, status: str):
@@ -498,14 +478,13 @@ class ChatBox(QtWidgets.QWidget):
         self.setMinimumWidth(500)
         self.setMaximumWidth(500)
 
-        self.chatName = getuser() + ": "
+        self.chatName = f"{getuser()}: "
 
         chatLayout = QtWidgets.QGridLayout()
         self.setLayout(chatLayout)
 
         chatLabel = QtWidgets.QLabel('Project Collaboration Chat')
-        chatLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
-        chatLabel.setAlignment(QtCore.Qt.AlignCenter)
+        chatLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         self.textView = QtWidgets.QPlainTextEdit()
         self.textView.setReadOnly(True)
         self.textSendBox = QtWidgets.QLineEdit()
@@ -533,14 +512,10 @@ class LoggingUpdateThread(QtCore.QThread):
         self.messageHandler = messageHandler
 
     def run(self):
-        while True:
-            if self.endLogging:
-                break
+        while not self.endLogging:
             if not self.messageHandler.logQueue.empty():
-                try:
+                with contextlib.suppress(queue.Empty):
                     logMsg = self.messageHandler.logQueue.get().getMessage()
                     self.loggingSignal.emit(logMsg)
-                except queue.Empty:
-                    pass
             else:
                 self.msleep(100)

Core/Interface/DockBarTwo.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 
+from pathlib import Path
+import magic
 from PySide6 import QtWidgets, QtCore, QtGui
-from Core.Interface import Stylesheets
+from Core.ResourceHandler import MinSizeStackedLayout, RichNotesEditor, resizePictureFromBuffer
+from Core.GlobalVariables import hidden_fields_dockbars
 
 
 class DockBarTwo(QtWidgets.QDockWidget):
@@ -12,12 +15,13 @@ class DockBarTwo(QtWidgets.QDockWidget):
         scrollAreaWidget = QtWidgets.QScrollArea()
 
         scrollAreaWidget.setWidget(self.entDetails)
-        scrollAreaWidget.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
+        scrollAreaWidget.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarPolicy.ScrollBarAsNeeded)
+        scrollAreaWidget.setWidgetResizable(True)
+        scrollAreaWidget.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
 
         childWidget.addTab(scrollAreaWidget, 'Entity Details')
         childWidget.addTab(self.oracle, 'Oracle')
-        self.setMaximumWidth(500)
-        self.resize(self.height(), 500)
+        childWidget.addTab(self.tabNotes, 'Notes')
 
     def __init__(self,
                  mainWindow,
@@ -26,11 +30,11 @@ class DockBarTwo(QtWidgets.QDockWidget):
                  title="DockBar Two"):
         super(DockBarTwo, self).__init__(parent=mainWindow)
 
-        self.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
-                             QtCore.Qt.RightDockWidgetArea)
-        self.setFeatures(QtWidgets.QDockWidget.DockWidgetMovable |
-                         QtWidgets.QDockWidget.DockWidgetFloatable |
-                         QtWidgets.QDockWidget.DockWidgetClosable)
+        self.setAllowedAreas(QtCore.Qt.DockWidgetArea.LeftDockWidgetArea |
+                             QtCore.Qt.DockWidgetArea.RightDockWidgetArea)
+        self.setFeatures(QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetMovable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetFloatable |
+                         QtWidgets.QDockWidget.DockWidgetFeature.DockWidgetClosable)
         self.setWindowTitle(title)
         self.resourceHandler = resourceHandler
         self.entityDB = entityDB
@@ -41,9 +45,39 @@ class DockBarTwo(QtWidgets.QDockWidget):
                                         self.entityDB,
                                         self.parent(),
                                         self)
+        self.tabNotes = TabNotesPanel(self)
 
         self.initialiseLayout()
 
+    def setNotesText(self, newText: str) -> None:
+        if not self.tabNotes.textEditor.isReadOnly():
+            self.tabNotes.textEditor.setReadOnly(True)
+        self.tabNotes.textEditor.contents = newText
+        self.tabNotes.textEditor.setMarkdown(newText)
+
+    def getNotesText(self) -> str:
+        """
+        Get text with the Markdown control characters.
+        @return:
+        """
+        return self.tabNotes.textEditor.toMarkdown()
+
+
+class TabNotesPanel(QtWidgets.QWidget):
+    """
+    This class is used to allow the user to take notes on a tab by tab
+    basis.
+    """
+
+    def __init__(self, parent=None):
+        super(TabNotesPanel, self).__init__(parent=parent)
+        tabNotesLayout = QtWidgets.QVBoxLayout()
+
+        self.textEditor = RichNotesEditor(self)
+
+        tabNotesLayout.addWidget(self.textEditor)
+        self.setLayout(tabNotesLayout)
+
 
 class EntityDetails(QtWidgets.QWidget):
     """
@@ -62,20 +96,21 @@ class EntityDetails(QtWidgets.QWidget):
         self.mainWindow = mainWindow
         self.resourceHandler = resourceHandler
         self.entityDB = entityDB
-        self.detailsLayout = QtWidgets.QStackedLayout()
+        self.detailsLayout = MinSizeStackedLayout()
         self.setLayout(self.detailsLayout)
+        self.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
 
-        layoutNothingSelected = QtWidgets.QGridLayout()
+        layoutNothingSelected = QtWidgets.QVBoxLayout()
         widgetNothing = QtWidgets.QWidget()
         widgetNothing.setLayout(layoutNothingSelected)
         self.detailsLayout.addWidget(widgetNothing)
 
-        layoutOneNodeSelected = QtWidgets.QGridLayout()
+        layoutOneNodeSelected = QtWidgets.QVBoxLayout()
         widgetOneNode = QtWidgets.QWidget()
         widgetOneNode.setLayout(layoutOneNodeSelected)
         self.detailsLayout.addWidget(widgetOneNode)
 
-        layoutMultipleItemsSelected = QtWidgets.QGridLayout()
+        layoutMultipleItemsSelected = QtWidgets.QVBoxLayout()
         widgetMultiItems = QtWidgets.QWidget()
         widgetMultiItems.setLayout(layoutMultipleItemsSelected)
         self.detailsLayout.addWidget(widgetMultiItems)
@@ -83,8 +118,7 @@ class EntityDetails(QtWidgets.QWidget):
         # Need to keep track of how many nodes are selected.
         # ~ Nothing Selected/Hovered Layout
         nothingLabel = QtWidgets.QLabel("Nothing is Selected.")
-        nothingLabel.setAlignment(QtCore.Qt.AlignHCenter |
-                                  QtCore.Qt.AlignVCenter)
+        nothingLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         layoutNothingSelected.addWidget(nothingLabel)
         ###
 
@@ -93,45 +127,49 @@ class EntityDetails(QtWidgets.QWidget):
         summaryLayout = QtWidgets.QGridLayout()
         summaryPanel = QtWidgets.QWidget()
         summaryPanel.setLayout(summaryLayout)
-        self.summaryIcon = QtWidgets.QLabel("")  # QtGui.QIcon(None)
+        self.summaryIcon = QtWidgets.QLabel("")
+        self.summaryIcon.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
         self.entityTypeLabel = QtWidgets.QLabel("")
-        self.entityUIDLabel = QtWidgets.QLabel("")
-        self.entityPrimaryLabel = QtWidgets.QLabel("")
+        self.entityUIDLabel = QtWidgets.QLineEdit("")
+        self.entityUIDLabel.setReadOnly(True)
+        self.entityPrimaryLabel = QtWidgets.QLineEdit("")
+        self.entityPrimaryLabel.setReadOnly(True)
         summaryLayout.addWidget(self.summaryIcon, 0, 0)
         summaryLayout.addWidget(self.entityTypeLabel, 0, 1, 1, 3)
         summaryLayout.addWidget(self.entityPrimaryLabel, 1, 1)
         summaryLayout.addWidget(self.entityUIDLabel, 2, 1)
-        layoutOneNodeSelected.addWidget(summaryPanel, 0, 0)
+        layoutOneNodeSelected.addWidget(summaryPanel, 1)
 
         # ~~ Part 2
         self.nodeLinkL = QtWidgets.QStackedLayout()
         nodeLinkSwitcher = QtWidgets.QWidget()
         nodeLinkSwitcher.setLayout(self.nodeLinkL)
-        layoutOneNodeSelected.addWidget(nodeLinkSwitcher, 1, 0)
+        layoutOneNodeSelected.addWidget(nodeLinkSwitcher, 1)
 
-        relationshipsLayout = QtWidgets.QGridLayout()
+        relationshipsLayout = QtWidgets.QVBoxLayout()
         relationshipsPanel = QtWidgets.QWidget()
         relationshipsPanel.setLayout(relationshipsLayout)
-        self.relationshipsIncomingTable = RelationshipsTable(self, mainWindow)
-        self.relationshipsOutgoingTable = RelationshipsTable(self, mainWindow)
+        self.relationshipsIncomingTable = RelationshipsTable(self, mainWindow, uidLabel=self.entityPrimaryLabel,
+                                                             incomingOrOutgoing=0)
+        self.relationshipsOutgoingTable = RelationshipsTable(self, mainWindow, uidLabel=self.entityPrimaryLabel,
+                                                             incomingOrOutgoing=1)
         self.relationshipsIncomingTable.setHeaderLabel('Incoming Links')
         self.relationshipsOutgoingTable.setHeaderLabel('Outgoing Links')
-        relationshipsLayout.addWidget(self.relationshipsIncomingTable, 0, 0)
-        relationshipsLayout.addWidget(self.relationshipsOutgoingTable, 1, 0)
+        relationshipsLayout.addWidget(self.relationshipsIncomingTable)
+        relationshipsLayout.addWidget(self.relationshipsOutgoingTable)
         self.nodeLinkL.addWidget(relationshipsPanel)
+        self.relationshipsIncomingTable.setMinimumHeight(125)
+        self.relationshipsOutgoingTable.setMinimumHeight(125)
 
         oneLinkRelLayout = QtWidgets.QHBoxLayout()
         oneLinkRelPanel = QtWidgets.QWidget()
         oneLinkRelPanel.setMaximumHeight(150)
         oneLinkRelPanel.setLayout(oneLinkRelLayout)
-        self.linkParent = SingleLinkItem(self, mainWindow)  # QtWidgets.QLabel("")
-        self.linkParent.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
+        self.linkParent = SingleLinkItem(self, mainWindow)
         self.linkIcon = QtWidgets.QLabel("")
         self.linkIcon.setMaximumHeight(90)
-        self.linkIcon.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
-        self.linkIcon.setAlignment(QtCore.Qt.AlignCenter)
-        self.linkChild = SingleLinkItem(self, mainWindow)  # QtWidgets.QLabel("")
-        self.linkChild.setStyleSheet(Stylesheets.DOCK_BAR_TWO_LINK)
+        self.linkIcon.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.linkChild = SingleLinkItem(self, mainWindow)
         oneLinkRelLayout.addWidget(self.linkParent)
         oneLinkRelLayout.addWidget(self.linkIcon)
         oneLinkRelLayout.addWidget(self.linkChild)
@@ -140,32 +178,29 @@ class EntityDetails(QtWidgets.QWidget):
         # ~~ Part 3
         scroll = QtWidgets.QScrollArea()
 
-        self.detailsLayoutOneNode = QtWidgets.QFormLayout()
+        self.detailsLayoutOneNode = QtWidgets.QGridLayout()
         detailsLayoutOneNodePanel = QtWidgets.QWidget()
         detailsLayoutOneNodePanel.setLayout(self.detailsLayoutOneNode)
         scroll.setWidget(detailsLayoutOneNodePanel)
         scroll.setWidgetResizable(True)
         scroll.setMinimumHeight(200)
-        # scroll.setMaximumHeight(400)
-        layoutOneNodeSelected.addWidget(scroll, 2, 0)
+        layoutOneNodeSelected.addWidget(scroll, 10)
 
         # ~ Multiple Links/Nodes Selected Layout
         self.nodesTable = RelationshipsTable(self, mainWindow)
-        self.nodesTable.setMaximumHeight(300)
+        self.nodesTable.setMaximumHeight(500)
         self.nodesTable.setHeaderLabels(["Entity", "Incoming Links", "Outgoing Links"])
         self.linksTable = LinksTable(self, mainWindow)
-        self.linksTable.setMaximumHeight(300)
-        self.linksTable.setHeaderLabels(["Parent", "Child"])
-        multiNodesTableLabelOne = QtWidgets.QLabel("Selected Nodes:")
-        layoutMultipleItemsSelected.addWidget(multiNodesTableLabelOne, 0, 0)
-        layoutMultipleItemsSelected.addWidget(self.nodesTable, 1, 0)
-        multiNodesTableLabelTwo = QtWidgets.QLabel("Selected Links:")
-        layoutMultipleItemsSelected.addWidget(multiNodesTableLabelTwo, 2, 0)
-        layoutMultipleItemsSelected.addWidget(self.linksTable, 3, 0)
-
-        self.setMinimumWidth(475)
-        self.setMaximumWidth(475)
-        self.setMaximumHeight(550)
+        self.linksTable.setMaximumHeight(500)
+        self.linksTable.setHeaderLabels(["Resolution", "Parent", "Child"])
+        self.multiNodesTableLabelOne = QtWidgets.QLabel("Selected Nodes:")
+        self.multiNodesTableLabelOne.setMaximumHeight(20)
+        layoutMultipleItemsSelected.addWidget(self.multiNodesTableLabelOne)
+        layoutMultipleItemsSelected.addWidget(self.nodesTable)
+        self.multiNodesTableLabelTwo = QtWidgets.QLabel("Selected Links:")
+        self.multiNodesTableLabelTwo.setMaximumHeight(20)
+        layoutMultipleItemsSelected.addWidget(self.multiNodesTableLabelTwo)
+        layoutMultipleItemsSelected.addWidget(self.linksTable)
 
         self.currentlyShown = []
 
@@ -180,39 +215,56 @@ class EntityDetails(QtWidgets.QWidget):
         self.currentlyShown = jsonDicts
         numberOfItems = len(jsonDicts)
         if numberOfItems == 0:
-            self.layout().setCurrentIndex(0)
+            self.detailsLayout.setCurrentIndex(0)
             return
         if None in jsonDicts:
             self.mainWindow.MESSAGEHANDLER.error('Received None value when trying to display widget details.',
                                                  popUp=False)
+            self.detailsLayout.setCurrentIndex(0)
             return
         self.clearDetailsHelper()
         isNode = None
-        if numberOfItems == 1:
-            entity = jsonDicts[0]
-            isNode = self.entityDB.isNode(entity['uid'])
-            self.populateOneSummaryHelper(entity, isNode)
-            self.populateOneDetailsHelper(entity)
-            if isNode:
-                self.populateOneRelationshipHelper(entity['uid'])
-            else:
-                self.populateOneLinkRelationshipHelper(entity['uid'])
-        else:
-            for item in jsonDicts:
-                if item.get('Entity Type') is not None:
-                    self.populateMultiRelationshipHelperNode(item)
+        try:
+            if numberOfItems == 1:
+                entity = jsonDicts[0]
+                isNode = self.entityDB.isNode(entity['uid'])
+                self.populateOneSummaryHelper(entity, isNode)
+                self.populateOneDetailsHelper(entity)
+                if isNode:
+                    self.populateOneRelationshipHelper(entity['uid'])
                 else:
-                    self.populateMultiRelationshipHelperLink(item)
-        self.switchLayoutHelper(numberOfItems, isNode)
+                    self.populateOneLinkRelationshipHelper(entity['uid'])
+            else:
+                nodesNumber = 0
+                linksNumber = 0
+                for item in jsonDicts:
+                    if isinstance(item['uid'], str):
+                        self.populateMultiRelationshipHelperNode(item)
+                        nodesNumber += 1
+                    else:
+                        self.populateMultiRelationshipHelperLink(item)
+                        linksNumber += 1
+                self.multiNodesTableLabelOne.setText(f'Selected Nodes:   {str(nodesNumber)}')
+                self.multiNodesTableLabelTwo.setText(f'Selected Links:   {str(linksNumber)}')
+
+            self.switchLayoutHelper(numberOfItems, isNode)
+        except Exception as exc:
+            # If an error is thrown at some point during the process, show the default nothing selected screen.
+            self.detailsLayout.setCurrentIndex(0)
+            self.mainWindow.MESSAGEHANDLER.error(f'Error occurred while trying to display the details of the selected '
+                                                 f'nodes: {str(exc)}', popUp=False, exc_info=False)
 
     # Display helper functions
     def clearDetailsHelper(self) -> None:
         """
         Clear tree widget items from tree widgets.
         """
-        rows = self.detailsLayoutOneNode.rowCount()
-        for row in range(rows):
-            self.detailsLayoutOneNode.removeRow(0)
+        itemToRemove = self.detailsLayoutOneNode.takeAt(0)
+        while itemToRemove is not None:
+            itemToRemove.widget().deleteLater()
+            del itemToRemove
+            itemToRemove = self.detailsLayoutOneNode.takeAt(0)
+
         self.relationshipsIncomingTable.clear()
         self.relationshipsOutgoingTable.clear()
         self.nodesTable.clear()
@@ -221,15 +273,44 @@ class EntityDetails(QtWidgets.QWidget):
     def populateOneDetailsHelper(self, jsonDict) -> None:
         if jsonDict is None or jsonDict == []:
             return
+        rowCount = 0
         for key in jsonDict:
-            if key == "uid" or key == "Child UIDs" or key == "Icon":
+            if key in hidden_fields_dockbars:
                 continue
             elif key == "Notes":
-                notesTextArea = QtWidgets.QPlainTextEdit(jsonDict[key])
-                notesTextArea.setReadOnly(True)
-                self.detailsLayoutOneNode.addRow(key, notesTextArea)
+                notesTextArea = RichNotesEditor(self, jsonDict[key], False)
+                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
+                self.detailsLayoutOneNode.addWidget(notesTextArea, rowCount, 1, 10, 1)
+                rowCount += 9
             else:
-                self.detailsLayoutOneNode.addRow(key, QtWidgets.QLabel(str(jsonDict[key])))
+                valueLabel = QtWidgets.QLineEdit(str(jsonDict[key]))
+                valueLabel.setReadOnly(True)
+
+                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
+                self.detailsLayoutOneNode.addWidget(valueLabel, rowCount, 1)
+            rowCount += 1
+        filePath = jsonDict.get('File Path')
+        if filePath is not None:
+            fullFilePath = Path(self.mainWindow.SETTINGS.value('Project/FilesDir')) / filePath
+            if fullFilePath.is_file():
+                magicType = magic.from_file(str(fullFilePath), mime=True)
+                if magicType.split('/')[0] == 'image':
+                    previewImage = QtGui.QImage(fullFilePath)
+                    if previewImage.isNull():
+                        # If launched from terminal, expect errors like the following to show up:
+                        # qt.gui.imageio: QImageIOHandler: Rejecting image as it exceeds the current allocation
+                        #   limit of 128 megabytes
+                        # This is fine - we are handling this here, by having the label show text instead of a null
+                        #   image.
+                        previewLabel = QtWidgets.QLabel('Error Creating Image Preview.')
+                    else:
+                        previewPixmap = QtGui.QPixmap(previewImage)
+                        previewLabel = QtWidgets.QLabel()
+                        previewLabel.setPixmap(previewPixmap.scaled(250,
+                                                                    250,
+                                                                    QtCore.Qt.AspectRatioMode.KeepAspectRatio))
+                    self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel('Preview:'), rowCount, 0)
+                    self.detailsLayoutOneNode.addWidget(previewLabel, rowCount, 1, 10, 1)
 
     def populateOneLinkRelationshipHelper(self, uid) -> None:
         first = uid[0]
@@ -237,9 +318,9 @@ class EntityDetails(QtWidgets.QWidget):
         second = uid[1]
         secondJson = self.entityDB.getEntity(second)
         firstPixmap = QtGui.QPixmap()
-        firstPixmap.loadFromData(firstJson.get('Icon'))
+        firstPixmap.loadFromData(resizePictureFromBuffer(firstJson.get('Icon'), (40, 40)))
         secondPixmap = QtGui.QPixmap()
-        secondPixmap.loadFromData(secondJson.get('Icon'))
+        secondPixmap.loadFromData(resizePictureFromBuffer(secondJson.get('Icon'), (40, 40)))
         self.linkParent.linkItemPic.setPixmap(firstPixmap)
         self.linkParent.linkItemName.setText(firstJson[list(firstJson)[1]])
         self.linkParent.linkItemUid = firstJson['uid']
@@ -254,26 +335,28 @@ class EntityDetails(QtWidgets.QWidget):
             uid = edge[0]
             edgeJson = self.entityDB.getEntity(uid)
             nodePixmap = QtGui.QPixmap()
-            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            nodePixmap.loadFromData(resizePictureFromBuffer(edgeJson.get('Icon'), (40, 40)))
             ResolutionTreeWidgetEntity(self.relationshipsIncomingTable,
                                        nodePixmap,
                                        edgeJson[list(edgeJson)[1]],
                                        uid)
+        self.relationshipsIncomingTable.setHeaderLabel(f'Incoming Links:   {len(inc)}')
         for edge in out:
             uid = edge[1]
             edgeJson = self.entityDB.getEntity(uid)
             nodePixmap = QtGui.QPixmap()
-            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            nodePixmap.loadFromData(resizePictureFromBuffer(edgeJson.get('Icon'), (40, 40)))
             ResolutionTreeWidgetEntity(self.relationshipsOutgoingTable,
                                        nodePixmap,
                                        edgeJson[list(edgeJson)[1]],
                                        uid)
+        self.relationshipsOutgoingTable.setHeaderLabel(f'Outgoing Links:   {len(out)}')
 
     def populateMultiRelationshipHelperNode(self, nodeJson) -> None:
         inc = len(self.entityDB.getIncomingLinks(nodeJson['uid']))
         out = len(self.entityDB.getOutgoingLinks(nodeJson['uid']))
         nodePixmap = QtGui.QPixmap()
-        nodePixmap.loadFromData(nodeJson.get('Icon'))
+        nodePixmap.loadFromData(resizePictureFromBuffer(nodeJson.get('Icon'), (40, 40)))
         ResolutionTreeWidgetEntity(self.nodesTable,
                                    nodePixmap,
                                    nodeJson[list(nodeJson)[1]],
@@ -287,6 +370,7 @@ class EntityDetails(QtWidgets.QWidget):
         child = self.entityDB.getEntity(uid[1])
         LinksTreeWidgetEntity(self.linksTable,
                               QtGui.QPixmap(self.resourceHandler.getLinkPicture()),
+                              linkJson['Resolution'],
                               parent[list(parent)[1]],
                               child[list(child)[1]],
                               uid)
@@ -297,7 +381,7 @@ class EntityDetails(QtWidgets.QWidget):
             self.entityPrimaryLabel.setText(jsonDict[list(jsonDict)[0]])
             self.entityTypeLabel.setText(jsonDict['Entity Type'])
             summaryPixmap = QtGui.QPixmap()
-            summaryPixmap.loadFromData(jsonDict.get('Icon'))
+            summaryPixmap.loadFromData(resizePictureFromBuffer(jsonDict.get('Icon'), (40, 40)))
             self.summaryIcon.setPixmap(summaryPixmap)
         else:
             self.entityUIDLabel.setText("--")
@@ -334,11 +418,12 @@ class SingleLinkItem(QtWidgets.QWidget):
         super().__init__(parent=parent)
 
         self.linkItemPic = QtWidgets.QLabel()
-        
-        self.linkItemPic.setAlignment(QtCore.Qt.AlignCenter)
-        self.linkItemName = QtWidgets.QLabel()
-        
-        self.linkItemName.setAlignment(QtCore.Qt.AlignCenter)
+
+        self.linkItemPic.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        self.linkItemName = QtWidgets.QLineEdit()
+        self.linkItemName.setReadOnly(True)
+
+        self.linkItemName.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         self.linkItemUid = ""
         self.setMaximumHeight(90)
 
@@ -358,27 +443,41 @@ class SingleLinkItem(QtWidgets.QWidget):
 
 class RelationshipsTable(QtWidgets.QTreeWidget):
 
-    def __init__(self, parent, mainWindow):
+    def __init__(self, parent, mainWindow, uidLabel: QtWidgets.QLabel = None, incomingOrOutgoing: int = None):
         super().__init__(parent=parent)
+
         self.mainWindow = mainWindow
+        self.incomingOrOutgoing = incomingOrOutgoing
+        self.uidLabel = uidLabel
 
     def mousePressEvent(self, event):
         """
         Have the canvas select the clicked item.
         """
+        # The super() call is not technically needed.
+        super().mousePressEvent(event)
         if event.button() == QtGui.Qt.MouseButton.LeftButton:
             itemClicked = self.itemAt(event.pos())
             if itemClicked is None:
                 return
-            self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
-        else:
-            super().mousePressEvent(event)
+            if self.uidLabel is not None:
+                potentialSelectedEntityUID = self.uidLabel.text()
+                if potentialSelectedEntityUID == "":
+                    self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
+                elif self.incomingOrOutgoing is not None:
+                    if self.incomingOrOutgoing == 0:
+                        self.mainWindow.setCurrentCanvasSelection([(itemClicked.entityUID, potentialSelectedEntityUID)])
+                    else:
+                        self.mainWindow.setCurrentCanvasSelection([(potentialSelectedEntityUID, itemClicked.entityUID)])
+            else:
+                self.mainWindow.setCurrentCanvasSelection([itemClicked.entityUID])
 
 
 class LinksTable(QtWidgets.QTreeWidget):
 
     def __init__(self, parent, mainWindow):
         super().__init__(parent=parent)
+
         self.mainWindow = mainWindow
 
     def mousePressEvent(self, event):
@@ -398,13 +497,15 @@ class LinksTreeWidgetEntity(QtWidgets.QTreeWidgetItem):
     def __init__(self,
                  parent,
                  icon=None,
+                 resolutionText=None,
                  ent1=None,
                  ent2=None,
                  uid=None):
         super().__init__(parent)
 
-        self.setText(0, str(ent1))
-        self.setText(1, str(ent2))
+        self.setText(0, resolutionText)
+        self.setText(1, str(ent1))
+        self.setText(2, str(ent2))
         self.uid = uid
         self.setIcon(0, icon)
         self.setIcon(1, icon)
@@ -444,9 +545,7 @@ class Oracle(QtWidgets.QWidget):
         self.setLayout(oracleLayout)
 
         self.answerLabel = QtWidgets.QLabel("Answer Section")
-        self.answerLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
-        self.answerLabel.setAlignment(QtCore.Qt.AlignHCenter |
-                                      QtCore.Qt.AlignVCenter)
+        self.answerLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         oracleLayout.addWidget(self.answerLabel, 1, 0, 1, 2)
         self.answerSection = QtWidgets.QPlainTextEdit()
         self.answerSection.setReadOnly(True)
@@ -456,9 +555,7 @@ class Oracle(QtWidgets.QWidget):
         oracleLayout.addWidget(self.answerSection, 2, 0, 1, 2)
 
         self.questionLabel = QtWidgets.QLabel("Ask a Question")
-        self.questionLabel.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
-        self.questionLabel.setAlignment(QtCore.Qt.AlignHCenter |
-                                        QtCore.Qt.AlignVCenter)
+        self.questionLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         oracleLayout.addWidget(self.questionLabel, 3, 0, 1, 2)
         self.questionSection = QtWidgets.QLineEdit()
         self.questionSection.setPlaceholderText("Ask a Question here.")

Core/Interface/Entity.py

@@ -1,42 +1,68 @@
 #!/usr/bin/env python3
 
+import contextlib
 from json import dumps
 import math
 from typing import Any, Optional
 
 from PySide6 import QtCore, QtGui, QtWidgets
 from PySide6.QtWidgets import QGraphicsItem
-from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem
+from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem, QGraphicsTextItem
 from PySide6.QtSvgWidgets import QGraphicsSvgItem
 
+from Core.ResourceHandler import resizePictureFromBuffer
+
+ENTITY_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
+LINK_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
+
 
 class BaseNode(QGraphicsItemGroup):
 
-    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str) -> None:
+    def __init__(self, pictureByteArray: QtCore.QByteArray, uid, primaryAttribute: str, font: QtGui.QFont,
+                 brush: QtGui.QBrush) -> None:
         super(BaseNode, self).__init__()
 
-        self.setCacheMode(self.DeviceCoordinateCache)
+        self.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+        resizedByteArray = resizePictureFromBuffer(pictureByteArray, (40, 40))
 
-        self.pixmapItem = QtGui.QPixmap()
-        self.pixmapItem.loadFromData(pictureByteArray)
-
-        if pictureByteArray.data().startswith(b'<svg '):
+        if pictureByteArray.data().startswith(b'<svg ') or pictureByteArray.data().startswith(b'<?xml'):
             self.iconItem = QGraphicsSvgItem()
-            self.iconItem.renderer().load(pictureByteArray)
+            self.iconItem.renderer().load(resizedByteArray)
             # Force recalculation of geometry, else this looks like 1 pixel.
             # https://stackoverflow.com/a/68182093
             self.iconItem.setElementId("")
         else:
-            self.iconItem = QGraphicsPixmapItem(self.pixmapItem)
+            pixmapItem = QtGui.QPixmap()
+            pixmapItem.loadFromData(resizedByteArray)
+            self.iconItem = QGraphicsPixmapItem(pixmapItem)
 
-        self.iconItem.setCacheMode(self.iconItem.DeviceCoordinateCache, QtCore.QSize(40, 40))
+        self.labelItem = QGraphicsTextItem('')
+        # Have to do it this way; directly assigning stuff does not work due to how PySide6 works.
+        labelDocument = self.labelItem.document()
+        textOption = labelDocument.defaultTextOption()
+        textOption.setWrapMode(QtGui.QTextOption.WrapMode.WrapAtWordBoundaryOrAnywhere)
+        textOption.setAlignment(QtCore.Qt.AlignmentFlag.AlignHCenter)
+        labelDocument.setDefaultTextOption(textOption)
+        self.labelItem.setDocument(labelDocument)
+
+        self.bannerIconItem = QGraphicsSvgItem()
+        self.bannerIconItem.setElementId("")
 
-        self.labelItem = QGraphicsSimpleTextItem('')
         self.addToGroup(self.iconItem)
         self.addToGroup(self.labelItem)
+        self.addToGroup(self.bannerIconItem)
+
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(ENTITY_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setDefaultTextColor(brush.color())
 
         self.updateLabel(primaryAttribute)
-        self.labelItem.setCacheMode(self.labelItem.DeviceCoordinateCache)
+
+        self.bannerIconItem.setPos(self.iconItem.x() + 15, self.iconItem.y() - 9)
+        self.bannerIconItem.setZValue(10)
 
         self.uid = uid
         self.setFlag(QGraphicsItem.ItemIsMovable, True)
@@ -46,22 +72,29 @@ class BaseNode(QGraphicsItemGroup):
         self.setAcceptHoverEvents(True)
 
         self.connectors = []
-        self.bookmarked = False
-        self.isBeingResolved = False
-        self.parentGroup = None
 
     def updateLabel(self, newText: str = '') -> None:
         if newText != '':
-            self.labelItem.setText(newText)
-            labelWidth = self.labelItem.boundingRect().width()
-            self.labelItem.setPos(self.iconItem.x() + 20 - (labelWidth / 2), self.iconItem.y() + 45)
+            if len(newText) > 50:
+                newText = f"{newText[:47]}..."
+            self.labelItem.setPlainText(newText)
+        self.labelItem.document().adjustSize()
+        self.labelItem.setPos(self.iconItem.x() + 20 - (self.labelItem.textWidth() / 2), self.iconItem.y() + 45)
+
+    def updateBanner(self, bannerHidden: bool = True, bannerGraphic: QtCore.QByteArray = None) -> None:
+        if bannerHidden:  # No icon visible
+            self.bannerIconItem.hide()
+            self.bannerIconItem.setVisible(False)
+            return
+        self.bannerIconItem.renderer().load(bannerGraphic)
+        self.bannerIconItem.show()
+        self.bannerIconItem.setVisible(True)
+        self.bannerIconItem.setElementId("")
 
     def removeConnector(self, connector) -> None:
         # Exception could be thrown if the connector is already deleted.
-        try:
+        with contextlib.suppress(ValueError):
             self.connectors.remove(connector)
-        except ValueError:
-            pass
 
     def addConnector(self, connector) -> None:
         self.connectors.append(connector)
@@ -84,23 +117,46 @@ class BaseNode(QGraphicsItemGroup):
         super().mouseDoubleClickEvent(event)
 
     def shape(self) -> QtGui.QPainterPath:
-        return self.iconItem.shape()
+        returnPath = QtGui.QPainterPath()
+        returnPath.addRect(QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80))
+        return returnPath
 
     def boundingRect(self) -> QtCore.QRectF:
-        return self.iconItem.boundingRect()
+        return QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80)
 
     def childrenBoundingRect(self) -> QtCore.QRectF:
-        return self.iconItem.boundingRect()
+        return QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80)
 
     def boundingRegion(self, itemToDeviceTransform: QtGui.QTransform) -> QtGui.QRegion:
         return self.iconItem.boundingRegion(itemToDeviceTransform)
 
+    def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
+              widget: Optional[QtWidgets.QWidget] = ...) -> None:
+        painter.setPen(QtCore.Qt.PenStyle.NoPen)
+        if self.scene().views()[0].zoom < self.scene().hideZoom:
+            # Looks stupid, but fixes bug where entities are deselected when zooming out past hideZoom level.
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
+        else:
+            self.labelItem.show()
+        if self.isSelected():
+            centerPoint = QtCore.QPointF(self.iconItem.x() + 20, self.iconItem.y() + 20)
+            selectionBackgroundGradient = QtGui.QRadialGradient(centerPoint, 80, centerPoint)
+            selectionBackgroundGradient.setColorAt(0.0, QtGui.QColor(250, 250, 255))
+            selectionBackgroundGradient.setColorAt(0.5, QtGui.QColor(231, 240, 253, 0))
+            painter.setBrush(selectionBackgroundGradient)
+            painter.drawRect(QtCore.QRectF(self.iconItem.x() - 20, self.iconItem.y() - 20, 80, 80))
+        super(BaseNode, self).paint(painter, option, widget)
+
 
 class GroupNode(BaseNode):
 
     # childNodes is a list of tuples, uid and picture, of all the nodes in the group.
-    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group') -> None:
-        super(GroupNode, self).__init__(pictureByteArray, uid, label)
+    def __init__(self, pictureByteArray, uid: str, label: str = 'Entity Group', font=None, brush=None) -> None:
+        super(GroupNode, self).__init__(pictureByteArray, uid, label, font, brush)
         self.groupedNodesConnectors = []
         self.itemsThatWereGrouped = []
         self.groupedNodesUid = set()
@@ -109,7 +165,7 @@ class GroupNode(BaseNode):
         self.listProxyWidget = None
 
     def itemChange(self, change: QtWidgets.QGraphicsItem.GraphicsItemChange, value: Any) -> Any:
-        if change == QtWidgets.QGraphicsItem.ItemSelectedChange:
+        if change == QtWidgets.QGraphicsItem.GraphicsItemChange.ItemSelectedChange:
             if value:
                 self.showList(None)
             else:
@@ -127,7 +183,7 @@ class GroupNode(BaseNode):
             except IndexError:
                 primaryField = ''
             iconPixmap = QtGui.QPixmap()
-            iconPixmap.loadFromData(entityJson['Icon'])
+            iconPixmap.loadFromData(resizePictureFromBuffer(entityJson['Icon'], (40, 40)))
 
             GroupNodeListItem(icon=iconPixmap, text=primaryField, uid=uid,
                               listview=self.listWidget.itemList)
@@ -149,7 +205,8 @@ class GroupNode(BaseNode):
     def formGroup(self, childNodeUIDs, listProxyWidget: QtWidgets.QGraphicsProxyWidget) -> None:
         [self.addItemToGroup(uid) for uid in childNodeUIDs]  # Should be faster than just a for loop
         self.listProxyWidget = listProxyWidget
-        self.listProxyWidget.setCacheMode(self.DeviceCoordinateCache)
+        self.listProxyWidget.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+        self.listProxyWidget.setZValue(100)
 
     def addItemToGroup(self, uid: str) -> None:
         self.groupedNodesUid.add(uid)
@@ -159,8 +216,13 @@ class GroupNode(BaseNode):
         incomingLinks = self.scene().parent().entityDB.getIncomingLinks(uid)
         outgoingLinks = self.scene().parent().entityDB.getOutgoingLinks(uid)
 
-        [self.scene().addLinkProgrammatic(link[0], self.uid) for link in incomingLinks]
-        [self.scene().addLinkProgrammatic(self.uid, link[1]) for link in outgoingLinks]
+        # Suppress exceptions, as these give false alarms when initially drawing the graph on the canvas.
+        for link in incomingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
+        for link in outgoingLinks:
+            self.scene().addLinkProgrammatic(link, self.scene().parent().entityDB.getLink(link)['Resolution'],
+                                             suppressNonExistentEntityException=True)
 
     def removeSpecificItemFromGroupIfExists(self, uid) -> bool:
         if uid in self.groupedNodesUid:
@@ -172,26 +234,28 @@ class GroupNode(BaseNode):
 
 # Ref: https://doc.qt.io/qt-5/qtwidgets-graphicsview-diagramscene-example.html#arrow-class-definition
 # Ref: https://github.com/PySide/Examples/blob/master/examples/graphicsview/diagramscene/diagramscene.py
-class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
+class BaseConnector(QGraphicsItemGroup):
 
-    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None) -> None:
+    def __init__(self, origin, destination, name: str = 'None', uid=None, parent=None,
+                 font: QtGui.QFont = None, brush: QtGui.QBrush = None) -> None:
         super(BaseConnector, self).__init__(parent)
-        self.setCacheMode(self.ItemCoordinateCache)
 
         self.myStartItem = origin
         self.myEndItem = destination
 
         self.labelItem = QGraphicsSimpleTextItem('')
-        self.labelItem.setCacheMode(self.labelItem.ItemCoordinateCache)
         self.addToGroup(self.labelItem)
+        if font is not None:
+            self.labelItem.setFont(font)
+        else:
+            self.labelItem.setFont(LINK_TEXT_FONT)
+        if brush is not None:
+            self.labelItem.setBrush(brush)
 
         self.updateLabel(name)
 
         if uid is not None:
-            if isinstance(uid, list) or isinstance(uid, set):
-                self.uid = set(uid)
-            else:
-                self.uid = {uid}
+            self.uid = set(uid) if isinstance(uid, (list, set)) else {uid}
         else:
             self.uid = {(origin.uid, destination.uid)}
 
@@ -203,21 +267,19 @@ class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
         self.myStartItem.addConnector(self)
         self.myEndItem.addConnector(self)
 
-        # Set as the wrong positions to force drawing.
-        self.oldStartPos = QtCore.QPointF(self.myStartItem.pos().x() + 1, 0)
-        self.oldEndPos = QtCore.QPointF(self.myEndItem.pos().x() + 1, 0)
-
         self.colorSelected = QtGui.QColor(0, 173, 238)
         self.colorDefault = QtGui.QColor(200, 200, 200)
         self.myColor = self.colorDefault
 
-        self.pen = QtGui.QPen(self.myColor, 2, QtCore.Qt.SolidLine,
-                              QtCore.Qt.RoundCap, QtCore.Qt.RoundJoin)
+        self.pen = QtGui.QPen(self.myColor, 2, QtCore.Qt.PenStyle.SolidLine,
+                              QtCore.Qt.PenCapStyle.RoundCap, QtCore.Qt.PenJoinStyle.RoundJoin)
 
         self.arrowHead = QtGui.QPolygonF()
         self.line = QtCore.QLineF()
 
     def updateLabel(self, newText: str = '') -> None:
+        if len(newText) > 50:
+            newText = f"{newText[:47]}..."
         self.labelItem.setText(newText)
         self.update()
 
@@ -240,12 +302,8 @@ class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
     def endItem(self) -> BaseNode:
         return self.myEndItem
 
-    def updatePosition(self) -> None:
-        self.line = QtCore.QLineF(self.mapFromItem(self.myStartItem, 0, 0), self.mapFromItem(self.myEndItem, 0, 0))
-        self.update()
-
     def boundingRect(self) -> QtCore.QRectF:
-        extra = (self.pen.width() + 20) / 2.0
+        extra = self.pen.width() + 20
         p1 = self.line.p1()
         p2 = self.line.p2()
         return QtCore.QRectF(p1, QtCore.QSizeF(p2.x() - p1.x(), p2.y() - p1.y())
@@ -260,35 +318,33 @@ class BaseConnector(QGraphicsItemGroup):  # TODO - Make letters bold?
     def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
               widget: Optional[QtWidgets.QWidget] = ...) -> None:
 
-        currentStartPos = self.myStartItem.pos()
-        currentEndPos = self.myEndItem.pos()
+        currentStartPos = self.myStartItem.pos() - self.pos()
+        currentEndPos = self.myEndItem.pos() - self.pos()
 
         self.myColor = self.colorSelected if self.isSelected() else self.colorDefault
 
-        if currentEndPos == self.oldEndPos and currentStartPos == self.oldStartPos:
-            myPen = QtGui.QPen(self.myColor)
-            painter.setPen(myPen)
-            painter.setBrush(self.myColor)
-            painter.drawLine(self.line)
-            painter.drawPolygon(self.arrowHead)
-            return
-
-        self.oldStartPos = currentStartPos
-        self.oldEndPos = currentEndPos
-
-        p1 = QtCore.QPointF(self.myStartItem.pos().x() + 20, self.myStartItem.pos().y() + 20)
-        p2 = QtCore.QPointF(self.myEndItem.pos().x() + 20, self.myEndItem.pos().y() + 20)
+        p1 = QtCore.QPointF(currentStartPos.x() + 20, currentStartPos.y() + 20)
+        p2 = QtCore.QPointF(currentEndPos.x() + 20, currentEndPos.y() + 20)
 
         line = QtCore.QLineF(p1, p2)
 
         if line.length() < 45:
-            self.labelItem.hide()
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
             return
 
         angle = math.atan2(line.dy(), - line.dx())
 
-        if line.length() < 50 + len(self.labelItem.text()) * 15:
-            self.labelItem.hide()
+        if (line.length() < 50 + len(self.labelItem.text()) * 15) or \
+                self.scene().views()[0].zoom < self.scene().hideZoom:
+            if self.isSelected():
+                self.labelItem.hide()
+                self.setSelected(True)
+            else:
+                self.labelItem.hide()
         else:
             self.labelItem.show()
             angle2 = math.degrees(math.pi - angle)
@@ -340,7 +396,7 @@ class GroupNodeChildList(QtWidgets.QWidget):
 
         self.setLayout(QtWidgets.QVBoxLayout())
         titleLabel = QtWidgets.QLabel('Child Items')
-        titleLabel.setAlignment(QtCore.Qt.AlignVCenter | QtCore.Qt.AlignHCenter)
+        titleLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         self.layout().addWidget(titleLabel)
 
         self.itemList = ChildListWidget()
@@ -352,9 +408,14 @@ class ChildListWidget(QtWidgets.QListWidget):
     def __init__(self) -> None:
         super(ChildListWidget, self).__init__()
         self.setAlternatingRowColors(True)
+        self.setSortingEnabled(True)
 
     def mouseMoveEvent(self, event: QtGui.QMouseEvent) -> None:
-        itemDragged = self.itemAt(event.pos())
+        super().mouseMoveEvent(event)
+
+        itemDragged = None
+        if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
+            itemDragged = self.itemAt(event.pos())
 
         if itemDragged is None:
             return
@@ -373,5 +434,3 @@ class ChildListWidget(QtWidgets.QListWidget):
             drag.setPixmap(pixmap)
         drag.setHotSpot(QtCore.QPoint(pixmap.rect().width() / 2, pixmap.rect().height() / 2))
         drag.exec_()
-
-        super().mousePressEvent(event)

Core/Interface/Stylesheets.py

@@ -1,228 +0,0 @@
-#!/usr/bin/env python3
-
-
-TOOLBAR_STYLESHEET = """QToolBar {background-color: rgb(41, 45, 62);
-                     color: rgba(248, 248, 242, 1) !important;
-                     font-family: Segoe UI;
-                     font-size: 13px;
-                     text-align: left;}
-                     
-                     QToolBar::separator {
-                     background-color: rgb(0, 173, 238);
-                     width: 3px;
-                     height: 3px;
-                     }
-                     """
-
-MAIN_WINDOW_STYLESHEET = """
-                        QWidget, QDialog{ 
-                        background-color: rgb(41, 45, 62);
-                         color: rgba(248, 248, 242, 1) !important;
-                         font-size: 15px;
-                         }
-                        QScrollBar:vertical {         
-                        background:rgb(44, 49, 58);
-                        width:7px;    
-                        margin: 0px 0px 0px 0px;
-                        }
-                        QScrollBar::handle:vertical {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
-                            min-height: 0px;
-                        }
-                        QScrollBar:horizontal {         
-                            background:rgb(44, 49, 58);
-                            height:7px;    
-                            margin: 0px 0px 0px 0px;
-                        }
-                        QScrollBar::handle:horizontal {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
-                        }
-
-                        QMenuBar { 
-                          color: #ffffff;
-                          background-color: rgb(33, 37, 43);
-                          font-size: 15px;
-                        }
-
-                        QPushButton {
-                        border: 2px solid rgb(0, 173, 238);
-                        border-radius: 6px;
-                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
-                        min-width: 50px;
-                        }
-
-                        QLabel { 
-                        border: 2px solid rgb(41, 45, 62);
-                        padding-left: 7px;
-                        border-left-color: rgb(0, 173, 238);
-                        }
-                        
-                        QLineEdit, QPlainTextEdit { 
-                        border: 0.5px solid rgb(0, 173, 238);
-                        border-radius: 6px;
-                        }
-                        
-                        QTabBar::tab {background: rgb(68, 66, 103);
-                        border: 2px solid rgb(41, 45, 62);
-                        border-radius: 3px;
-                        min-height: 3ex;
-                        }
-                        
-                        QTabBar::tab:selected {background: rgb(41, 45, 62);
-                        border: 2px solid rgb(41, 45, 62);
-                        min-height: 2.5ex;
-                        border-radius: 3px;
-                        border-top-color: rgb(0, 173, 238);
-                        }
-                        
-                        
-                        """
-
-DOCK_BAR_TWO_LINK = """
-                  QLabel { 
-                        border: 1px solid rgb(41, 45, 62);
-                        }
-                  """
-
-DOCK_BAR_LABEL = """
-                  QLabel { 
-                        border: 1px solid rgb(41, 45, 62);
-                        border-radius: 2px;
-                        border-bottom-color: rgb(0, 173, 238);
-                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                          stop: 1 rgb(0, 173, 238), stop: 0.85 rgb(41, 45, 62));
-                        }
-                  """
-
-DROPDOWN_MENU_STYLE = "background-color: rgb(41, 45, 62);" \
-                      "color: rgba(248, 248, 242, 1) !important;" \
-                      "border: 2px solid rgb(44, 49, 58);" \
-                      "border-left-color: rgb(189, 147, 249);" \
-                      "font-family: Segoe UI;" \
-                      "font-size: 13px;"
-
-TEXT_BOX_STYLESHEET = "QLineEdit, QLabel{" \
-                      "border-bottom: 1px solid rgb(0, 173, 238);" \
-                      "}"
-
-CHECK_BOX_STYLESHEET = "QCheckBox::indicator:unchecked" \
-                       "{" \
-                       "border: 0.5px solid rgb(0, 173, 238);" \
-                       "background: none;" \
-                       "}"
-
-RADIO_BUTTON_STYLESHEET = "QRadioButton::indicator:unchecked" \
-                          "{" \
-                          "border: 0.5px solid rgb(0, 173, 238);" \
-                          "background: none;" \
-                          "border-radius:          7px;" \
-                          "}"
-
-SETTINGS_WIDGET_STYLESHEET = "#settingsWidget {background-color:rgb(41, 45, 62);}"
-
-MENUS_STYLESHEET = "background-color: rgb(41, 45, 62);" \
-                   "color: rgba(248, 248, 242, 1) !important;" \
-                   "border: 1px solid rgb(44, 49, 58);" \
-                   "border-bottom: 1px solid rgb(0, 173, 238);" \
-                   "font-family: Segoe UI;" \
-                   "font-size: 13px;" \
-                   "text-align: left;"
-
-MENUS_STYLESHEET_2 = """QMenu::item{
-                         background-color: rgb(41, 45, 62);
-                         color: rgba(248, 248, 242, 1) !important;
-                         border-left: 1px solid rgb(0, 173, 238);
-                         padding-right: 4px;
-                         padding-bottom: 4px;
-                         padding-top: 4px;
-                         font-size: 15px;
-                         text-align: left;
-                         padding-left: 4px;
-                         }
-                        
-                         QMenu::item:selected{
-                         background-color: rgb(0, 85, 127);
-                         color: rgb(255, 255, 255);
-                         }
-                         
-                         QMenu::item:disabled {
-                         background-color:rgb(81, 87, 114);
-                         }"""
-
-MERGE_STYLESHEET = """
-                QWidget, QDialog{ 
-                    background-color: rgb(41, 45, 62);
-                    color: rgba(248, 248, 242, 1) !important;
-                    font-family: Segoe UI;
-                    font-size: 13px;}
-                   
-                   QScrollBar:vertical {         
-                        background:rgb(44, 49, 58);
-                        width:7px;    
-                        margin: 0px 0px 0px 0px;
-                        }
-                        QScrollBar::handle:vertical {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
-                            min-height: 0px;
-                        }
-                        QScrollBar:horizontal {         
-                            background:rgb(44, 49, 58);
-                            height:7px;    
-                            margin: 0px 0px 0px 0px;
-                        }
-                        QScrollBar::handle:horizontal {
-                            background: qlineargradient(x1:0, y1:0, x2:1, y2:0,
-                            stop: 0 rgb(103, 110, 149), stop: 0.5 rgb(103, 110, 149), stop:1 rgb(103, 110, 149));
-                        }"""
-
-RESOLUTION_WIZARD_STYLESHEET = "background-color: rgb(41, 45, 62);" \
-                               "color: rgba(248, 248, 242, 1) !important;" \
-                               "padding-bottom: 5px;" \
-                               "font-family: Segoe UI;" \
-                               "font-size: 13px;"
-
-PATH_INPUT_STYLESHEET = "border: 2px solid rgb(44, 49, 58);" \
-                        "border-radius: 25px;" \
-                        "padding: 4px; " \
-                        "background-color: rgb(129, 133, 137);" \
-                        "color: rgba(248, 248, 242, 1) !important;"
-
-BUTTON_STYLESHEET = """
-                        QPushButton {
-                        border: 2px solid rgb(0, 173, 238);
-                        border-radius: 6px;
-                        color: rgba(248, 248, 242, 1) !important;
-                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
-                        min-width: 80px;
-                        }
-                        QPushButton:pressed {
-                            background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                              stop: 0 #dadbde, stop: 1 #f6f7fa);
-                        }
-                    """
-
-BUTTON_STYLESHEET_2 = """
-                        QPushButton {
-                        border: 2px solid rgb(0, 173, 238);
-                        border-radius: 6px;
-                        color: rgba(248, 248, 242, 1) !important;
-                        background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                          stop: 1 rgb(0, 173, 238), stop: 0.7 rgb(44, 49, 58));
-                        
-                        min-width: 300px;
-                        
-                        }
-                        QPushButton:pressed {
-                            background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
-                                                              stop: 0 #dadbde, stop: 1 #f6f7fa);
-                        }
-                    """
-
-SELECT_PROJECT_STYLESHEET = "background-color: rgb(41, 45, 62);" \
-                            "color: rgba(248, 248, 242, 1) !important;" \
-                            "border-color: rgb(0, 173, 238);"

Core/Interface/ToolBarOne.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 from PySide6 import QtWidgets, QtGui
-from Core.Interface import Stylesheets
 
 
 class ToolBarOne(QtWidgets.QToolBar):
@@ -10,14 +9,14 @@ class ToolBarOne(QtWidgets.QToolBar):
         # Parent is (expected to be) mainWindow.
         super().__init__(title, parent=parent)
         self.setObjectName(title)
-        self.setToolButtonStyle(QtGui.Qt.ToolButtonTextUnderIcon)
-        self.setStyleSheet(Stylesheets.TOOLBAR_STYLESHEET)
+        self.setToolButtonStyle(QtGui.Qt.ToolButtonStyle.ToolButtonTextUnderIcon)
 
         newCanvas = QtGui.QAction("Add Canvas",
                                   self,
                                   statusTip="Create new Canvas or Open existing Canvas",
                                   triggered=self.addCanvas,
                                   icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('addCanvas')))
+        newCanvas.setShortcut('Ctrl+N')
         self.addAction(newCanvas)
         self.insertSeparator(newCanvas)
 
@@ -92,13 +91,13 @@ class ToolBarOne(QtWidgets.QToolBar):
         self.addAction(splitEntity)
         self.insertSeparator(splitEntity)
 
-        generateReport = QtGui.QAction('Generate Report',
-                                       self,
-                                       statusTip="Generate Report of selected entities.",
-                                       triggered=self.generateReports,
-                                       icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
-        self.addAction(generateReport)
-        self.insertSeparator(generateReport)
+        generateReportAction = QtGui.QAction('Generate Report',
+                                             self,
+                                             statusTip="Generate Report from the selected nodes.",
+                                             triggered=self.generateReport,
+                                             icon=QtGui.QIcon(self.parent().RESOURCEHANDLER.getIcon('generateReport')))
+        self.addAction(generateReportAction)
+        self.insertSeparator(generateReportAction)
 
         rearrangeCanvas = QtGui.QAction('Rearrange Canvas',
                                         self,
@@ -120,14 +119,6 @@ class ToolBarOne(QtWidgets.QToolBar):
     def addCanvas(self):
         self.parent().addCanvas()
 
-    def configureProxySettings(self):
-        pass
-
-    ###
-    def publishCanvas(self):
-        pass
-
-    ###
     def selectLeafNodes(self):
         self.parent().selectLeafNodes()
 
@@ -147,7 +138,7 @@ class ToolBarOne(QtWidgets.QToolBar):
         self.parent().uploadFiles()
 
     def downloadSelectedFiles(self):
-        self.parent().downloadFile()
+        self.parent().downloadFiles()
 
     def mergeEntities(self):
         self.parent().mergeEntities()
@@ -155,5 +146,5 @@ class ToolBarOne(QtWidgets.QToolBar):
     def splitEntity(self):
         self.parent().splitEntity()
 
-    def generateReports(self):
+    def generateReport(self):
         self.parent().generateReport()

Core/MessageHandler.py

@@ -18,11 +18,11 @@ class MessageHandler:
     """
 
     def debug(self, message, exc_info=True):
-        logging.debug(message, exc_info=exc_info)
+        self.linkScopeLogger.debug(message, exc_info=exc_info)
         return message
 
     def info(self, message, popUp=False, exc_info=False):
-        logging.info(message, exc_info=exc_info)
+        self.linkScopeLogger.info(message, exc_info=exc_info)
         if popUp:
             msgBox = QtWidgets.QMessageBox()
             QtWidgets.QMessageBox.information(msgBox,
@@ -31,7 +31,7 @@ class MessageHandler:
         return message
 
     def warning(self, message, popUp=False, exc_info=False):
-        logging.warning(message, exc_info=exc_info)
+        self.linkScopeLogger.warning(message, exc_info=exc_info)
         if popUp:
             msgBox = QtWidgets.QMessageBox()
             QtWidgets.QMessageBox.warning(msgBox,
@@ -39,8 +39,8 @@ class MessageHandler:
                                           self.mainWindow.tr(message))
         return message
 
-    def error(self, message, popUp=True, exc_info=True):
-        logging.error(message, exc_info=exc_info)
+    def error(self, message, popUp=True, exc_info=False):
+        self.linkScopeLogger.error(message, exc_info=exc_info)
         if popUp:
             msgBox = QtWidgets.QMessageBox()
             QtWidgets.QMessageBox.critical(msgBox,
@@ -49,7 +49,7 @@ class MessageHandler:
         return message
 
     def critical(self, message, popUp=True, exc_info=True):
-        logging.critical(message, exc_info=exc_info)
+        self.linkScopeLogger.critical(message, exc_info=exc_info)
         if popUp:
             msgBox = QtWidgets.QMessageBox()
             QtWidgets.QMessageBox.critical(msgBox,
@@ -59,28 +59,27 @@ class MessageHandler:
 
     # Set the severity level
     def setSeverityLevel(self, level):
-        currentLogLevel = logging.root.level
+        currentLogLevel = self.linkScopeLogger.level
         try:
             level = int(level)
-            logging.root.setLevel(level)
-            self.mainWindow.SETTINGS.setValue("Logging/Severity", logging.root.level)
+            self.linkScopeLogger.setLevel(level)
+            self.mainWindow.SETTINGS.setValue("Logging/Severity", self.linkScopeLogger.level)
         except ValueError:
             self.warning("Invalid Severity Level specified.")
-            logging.root.setLevel(currentLogLevel)
+            self.linkScopeLogger.setLevel(currentLogLevel)
             self.mainWindow.SETTINGS.setValue("Logging/Severity", currentLogLevel)
 
     def getSeverityLevel(self):
-        return logging.root.level
+        return self.linkScopeLogger.level
 
     def changeLogfile(self, newLogFile):
         self.mainWindow.SETTINGS.setValue("Logging/Logfile", newLogFile)
         self.logFileHandler = logging.FileHandler(
             self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
-        rootLogger = logging.getLogger()
-        for handler in rootLogger.handlers[:]:
+        for handler in self.linkScopeLogger.handlers[:]:
             if isinstance(handler, logging.FileHandler):
-                rootLogger.removeHandler(handler)
-        rootLogger.addHandler(self.logFileHandler)
+                self.linkScopeLogger.removeHandler(handler)
+        self.linkScopeLogger.addHandler(self.logFileHandler)
 
     def __init__(self, parentObject):
         self.mainWindow = parentObject
@@ -94,11 +93,15 @@ class MessageHandler:
         self.logFormatterQueue = logging.Formatter('[%(asctime)s] - %(levelname)s: %(message)s')
         self.logFileHandler.setFormatter(self.logFormatter)
         self.logQueueHandler.setFormatter(self.logFormatterQueue)
+
+        # Do not handle root logger messages
         rootLogger = logging.getLogger()
-        for handler in rootLogger.handlers[:]:
-            if isinstance(handler, logging.FileHandler):
+        for handler in rootLogger.handlers:
+            if isinstance(handler, logging.Handler):
                 rootLogger.removeHandler(handler)
-        rootLogger.addHandler(self.logFileHandler)
-        rootLogger.addHandler(self.logQueueHandler)
+
+        self.linkScopeLogger = logging.getLogger('LinkScope_Logger')
+        self.linkScopeLogger.addHandler(self.logFileHandler)
+        self.linkScopeLogger.addHandler(self.logQueueHandler)
 
         self.setSeverityLevel(self.mainWindow.SETTINGS.value("Logging/Severity", logging.INFO))

Core/PathHelper.py

@@ -46,7 +46,7 @@ def is_pathname_valid(pathname: str) -> bool:
         # environment variable); else, the typical root directory.
         root_dirname = os.environ.get('HOMEDRIVE', 'C:') \
             if sys.platform == 'win32' else os.path.sep
-        assert os.path.isdir(root_dirname)  # ...Murphy and her ironclad Law
+        assert os.path.isdir(root_dirname)  # ...Murphy and her ironclad Law  # nosec
 
         # Append a path separator to this directory if needed.
         root_dirname = root_dirname.rstrip(os.path.sep) + os.path.sep

Core/ReportGeneration.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
 
+import tempfile
+import re
+from shutil import rmtree
+from svglib.svglib import svg2rlg
+from uuid import uuid4
 from datetime import datetime
 from pathlib import Path
 from reportlab.lib import colors
@@ -10,12 +15,372 @@ from reportlab.platypus.tableofcontents import TableOfContents
 from reportlab.lib.units import cm
 
 from reportlab.pdfgen import canvas
-from reportlab.platypus import (Paragraph, PageBreak, Image, Spacer, Table,
-                                ParagraphAndImage)
+from reportlab.platypus import Paragraph, PageBreak, Image, Spacer, Table, ParagraphAndImage
 from reportlab.lib.enums import TA_LEFT, TA_CENTER, TA_JUSTIFY
 from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
 from reportlab.lib.pagesizes import LETTER, inch
 from reportlab.graphics.shapes import Line, Drawing
+from PySide6 import QtWidgets
+
+from Core.Interface.Entity import BaseNode
+from Core.GlobalVariables import avoid_parsing_fields
+
+
+class ReportWizard(QtWidgets.QWizard):
+    def __init__(self, parent):
+        super(ReportWizard, self).__init__(parent=parent)
+        self.reportTempFolder = tempfile.mkdtemp()
+
+        self.primaryFieldsList = []
+
+        self.addPage(InitialConfigPage(self))
+        self.addPage(TitlePage(self))
+
+        self.addPage(SummaryPage(self))
+
+        self.selectedNodes = [entity for entity in
+                              self.parent().centralWidget().tabbedPane.getCurrentScene().selectedItems()
+                              if isinstance(entity, BaseNode)]
+        for selectedNode in self.selectedNodes:
+            # used in wizard
+            self.primaryField = selectedNode.labelItem.toPlainText()
+            self.uid = selectedNode.uid
+
+            # used in report generation
+            self.primaryFieldsList.append(self.primaryField)
+            self.addPage(EntityPage(self))
+
+        self.setWizardStyle(QtWidgets.QWizard.WizardStyle.ModernStyle)
+        self.setWindowTitle("Generate Report Wizard")
+
+        self.button(QtWidgets.QWizard.WizardButton.FinishButton).clicked.connect(self.onFinish)
+
+    def onFinish(self):
+        outgoingEntitiesForEachEntity = []
+        incomingEntitiesForEachEntity = []
+        outgoingEntityPrimaryFieldsForEachEntity = []
+        incomingEntityPrimaryFieldsForEachEntity = []
+
+        entityList = []
+        reportData = []
+        for pageID in self.pageIds():
+            pageObject = self.page(pageID)
+            reportData.append(pageObject.getData())
+
+        for selectedNode in self.selectedNodes:
+            uid = selectedNode.uid
+            entityList.append(self.parent().LENTDB.getEntity(uid))
+            outgoing = self.parent().LENTDB.getOutgoingLinks(uid)
+            incoming = self.parent().LENTDB.getIncomingLinks(uid)
+
+            outgoingEntities = []
+            incomingEntities = []
+            outgoingNames = []
+            incomingNames = []
+
+            for out in outgoing:
+                outLink = self.parent().LENTDB.getLink(out)
+                outgoingEntities.append(outLink)
+                outgoingEntityJson = self.parent().LENTDB.getEntity(outLink['uid'][1])
+                outgoingNames.append(outgoingEntityJson[list(outgoingEntityJson)[1]])
+
+            for inc in incoming:
+                inLink = self.parent().LENTDB.getLink(inc)
+                incomingEntities.append(inLink)
+                incomingEntityJson = self.parent().LENTDB.getEntity(inLink['uid'][0])
+
+                incomingNames.append(incomingEntityJson[list(incomingEntityJson)[1]])
+
+            outgoingEntityPrimaryFieldsForEachEntity.append(outgoingNames)
+            incomingEntityPrimaryFieldsForEachEntity.append(incomingNames)
+            outgoingEntitiesForEachEntity.append(outgoingEntities)
+            incomingEntitiesForEachEntity.append(incomingEntities)
+
+        path = Path(reportData[0].get('SavePath'))
+
+        canvasName = reportData[2].get('CanvasName')
+        viewPortBool = reportData[2].get('ViewPort')
+
+        canvasPicture = self.parent().getPictureOfCanvas(canvasName, viewPortBool, True)
+        canvasImagePath = Path(self.reportTempFolder) / 'canvas.png'
+        canvasPicture.save(str(canvasImagePath), "PNG")
+
+        # timelinePicture = self.parent().dockbarThree.timeWidget.takePictureOfView(False)
+        # timelineImagePath = Path(temp_dir.name) / 'timeline.png'
+        # timelinePicture.save(str(timelineImagePath), "PNG")
+
+        savePath = Path(reportData[0]['SavePath']).absolute()
+
+        try:
+            PDFReport(str(path), reportData, outgoingEntitiesForEachEntity,
+                      incomingEntitiesForEachEntity, entityList, canvasImagePath, None,  # <timelinePic
+                      self.primaryFieldsList, incomingEntityPrimaryFieldsForEachEntity,
+                      outgoingEntityPrimaryFieldsForEachEntity)
+
+            self.parent().MESSAGEHANDLER.debug(reportData)
+            self.parent().MESSAGEHANDLER.info(
+                f"Saved Report at: {str(savePath)}", popUp=True
+            )
+        except PermissionError:
+            self.parent().MESSAGEHANDLER.error(
+                f"Could not generate report. No permission to save at the chosen location: {str(savePath)}",
+                popUp=True,
+                exc_info=False,
+            )
+        except Exception as exc:
+            self.parent().MESSAGEHANDLER.error(
+                f"Could not generate report: {str(exc)}", popUp=True, exc_info=True
+            )
+        finally:
+            rmtree(self.reportTempFolder)
+
+
+class InitialConfigPage(QtWidgets.QWizardPage):
+    def __init__(self, parent=None):
+        super(InitialConfigPage, self).__init__(parent)
+        self.subtitleLabel = QtWidgets.QLabel("Path to save the report at: ")
+        self.savePathEdit = QtWidgets.QLineEdit()
+        self.setTitle("Initial Configuration Wizard")
+
+        pDirButton = QtWidgets.QPushButton("Save Report As...")
+        pDirButton.clicked.connect(self.editPath)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(self.subtitleLabel)
+        hLayout.addWidget(self.savePathEdit)
+        hLayout.addWidget(pDirButton)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def editPath(self):
+        selectedPath = QtWidgets.QFileDialog.getSaveFileName(self,
+                                                             "File Name to Save As",
+                                                             str(Path.home()),
+                                                             filter="PDF Files (*.pdf)",
+                                                             options=QtWidgets.QFileDialog.Option.DontUseNativeDialog)
+        selectedPath = selectedPath[0]
+        if selectedPath != '':
+            savePath = Path(selectedPath).absolute()
+            if savePath.suffix != '.pdf':
+                savePath = savePath.with_suffix(f"{savePath.suffix}.pdf")
+            self.savePathEdit.setText(str(savePath))
+
+    def getData(self):
+        return {'SavePath': self.savePathEdit.text()}
+
+
+class TitlePage(QtWidgets.QWizardPage):
+    def __init__(self, parent=None):
+        super(TitlePage, self).__init__(parent)
+        self.inputTitleEdit = QtWidgets.QLineEdit()
+        self.inputSubtitleEdit = QtWidgets.QLineEdit()
+        self.inputAuthorsEdit = QtWidgets.QLineEdit()
+        self.setTitle("Title Page Wizard")
+
+        titleLabel = QtWidgets.QLabel("Title: ")
+        subtitleLabel = QtWidgets.QLabel("Subtitle: ")
+        authorsLabel = QtWidgets.QLabel("Authors: ")
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(titleLabel)
+        hLayout.addWidget(self.inputTitleEdit)
+        hLayout.addWidget(subtitleLabel)
+        hLayout.addWidget(self.inputSubtitleEdit)
+        hLayout.addWidget(authorsLabel)
+        hLayout.addWidget(self.inputAuthorsEdit)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def getData(self):
+        return {
+            'Title': self.inputTitleEdit.text(),
+            'Subtitle': self.inputSubtitleEdit.text(),
+            'Authors': self.inputAuthorsEdit.text(),
+        }
+
+
+class SummaryPage(QtWidgets.QWizardPage):
+    def __init__(self, parent):
+        super(SummaryPage, self).__init__(parent=parent.parent())
+        self.setTitle("Summary Page Wizard")
+        self.inputNotesEdit = QtWidgets.QPlainTextEdit()
+        self.canvasDropDownMenu = QtWidgets.QComboBox()
+        self.viewPortCheckBox = QtWidgets.QCheckBox('ViewPort Only')
+        self.viewPortCheckBox.setChecked(False)
+        self.canvasNames = list(self.parent().centralWidget().tabbedPane.canvasTabs.keys())
+
+        summaryLabel = QtWidgets.QLabel("Summary Notes: ")
+
+        canvasLabel = QtWidgets.QLabel("Select canvas to be displayed: ")
+        for canvasName in self.canvasNames:
+            self.canvasDropDownMenu.addItem(canvasName)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(summaryLabel)
+        hLayout.addWidget(self.inputNotesEdit)
+        hLayout.addWidget(canvasLabel)
+        hLayout.addWidget(self.viewPortCheckBox)
+        hLayout.addWidget(self.canvasDropDownMenu)
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def getData(self):
+        return {
+            'SummaryNotes': self.inputNotesEdit.toPlainText(),
+            'CanvasName': self.canvasDropDownMenu.currentText(),
+            'ViewPort': self.viewPortCheckBox.isChecked(),
+        }
+
+
+class EntityPage(QtWidgets.QWizardPage):
+    def __init__(self, parent: ReportWizard):
+        super(EntityPage, self).__init__(parent=parent.parent())
+        self.reportWizard = parent
+
+        self.setTitle("Entity Page Wizard")
+        self.setMinimumSize(300, 700)
+
+        self.entityName = parent.primaryField
+        self.entityUID = parent.uid
+
+        self.inputNotesEdit = QtWidgets.QPlainTextEdit()
+        self.inputImageEdit = QtWidgets.QLineEdit()
+        self.inputImageEdit.setReadOnly(True)
+        self.addAppendixButton = QtWidgets.QPushButton("Add New Appendix Section")
+        self.removeAppendixButton = QtWidgets.QPushButton("Remove Last Appendix Section")
+
+        self.scrolllayout = QtWidgets.QVBoxLayout()
+        self.scrollwidget = QtWidgets.QWidget()
+
+        self.defaultpic = self.parent().LENTDB.getEntity(self.entityUID).get('Icon')
+
+        summaryLabel = QtWidgets.QLabel(f"Entity {self.entityName} Notes: ")
+
+        imageLabel = QtWidgets.QLabel("Image Path: ")
+        pDirButton = QtWidgets.QPushButton("Select Image...")
+        pDirButton.clicked.connect(self.editPath)
+        pDirButton.setDisabled(True)
+        self.imageCheckBox = QtWidgets.QCheckBox('Add Custom Entity Image')
+        self.imageCheckBox.setChecked(False)
+        self.imageCheckBox.toggled.connect(pDirButton.setEnabled)
+
+        self.addAppendixButton.clicked.connect(self.addSection)
+        self.removeAppendixButton.clicked.connect(self.removeSection)
+
+        self.scrollwidget.setLayout(self.scrolllayout)
+
+        scroll = QtWidgets.QScrollArea()
+        scroll.setWidgetResizable(True)
+        scroll.setWidget(self.scrollwidget)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(summaryLabel)
+        hLayout.addWidget(self.inputNotesEdit)
+
+        hLayout.addWidget(self.imageCheckBox)
+        hLayout.addWidget(imageLabel)
+        hLayout.addWidget(self.inputImageEdit)
+        hLayout.addWidget(pDirButton)
+
+        hLayout.addItem(QtWidgets.QSpacerItem(10, 30))
+        hLayout.addWidget(self.addAppendixButton)
+        hLayout.addWidget(self.removeAppendixButton)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        layout.addWidget(scroll)
+
+        self.setLayout(layout)
+
+    def editPath(self) -> None:
+        selectedPath = QtWidgets.QFileDialog().getOpenFileName(parent=self, caption='Select New Icon',
+                                                               dir=str(Path.home()),
+                                                               options=QtWidgets.QFileDialog.Option.DontUseNativeDialog,
+                                                               filter="Image Files (*.png *.jpg)")[0]
+        if selectedPath != '':
+            self.inputImageEdit.setText(str(Path(selectedPath).absolute()))
+
+    def addSection(self) -> None:
+        appendixWidget = AppendixWidget()
+        self.scrolllayout.addWidget(appendixWidget)
+
+    def removeSection(self) -> None:
+        if numChildren := self.scrolllayout.count():
+            appendixItem = self.scrolllayout.takeAt(numChildren - 1)
+            appendixItem.widget().deleteLater()
+
+    def getData(self):
+        appendixNotes = []
+        if self.inputImageEdit.text() != '' and self.imageCheckBox.isChecked():
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': self.inputImageEdit.text()}
+        elif 'PNG' in str(self.defaultpic):
+            imagePath = Path(self.reportWizard.reportTempFolder) / f'{str(uuid4())}.png'
+            with open(imagePath, 'wb') as tempFile:
+                tempFile.write(bytearray(self.defaultpic.data()))
+
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': str(imagePath)}
+        else:
+            if 'svg' not in str(self.defaultpic):
+                # Default picture is an SVG.
+                self.defaultpic = self.reportWizard.parent().RESOURCEHANDLER.getEntityDefaultPicture(
+                    self.reportWizard.parent().LENTDB.getEntity(self.entityUID)['Entity Type'])
+            contents = bytearray(self.defaultpic)
+            widthRegex = re.compile(b' width="\d*" ')
+            for widthMatches in widthRegex.findall(self.defaultpic):
+                contents = contents.replace(widthMatches, b' ')
+            heightRegex = re.compile(b' height="\d*" ')
+            for heightMatches in heightRegex.findall(self.defaultpic):
+                contents = contents.replace(heightMatches, b' ')
+            contents = contents.replace(b'<svg ', b'<svg height="150" width="150" ')
+
+            imagePath = Path(self.reportWizard.reportTempFolder) / f'{str(uuid4())}.svg'
+            with open(imagePath, 'wb') as tempFile:
+                tempFile.write(contents)
+
+            image = svg2rlg(imagePath)
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': image}
+
+        for index in range(self.scrolllayout.count()):
+            childWidget = self.scrolllayout.itemAt(index).widget()
+            appendixDict = {'AppendixEntityNotes': childWidget.inputAppendixNotesEdit.toPlainText(),
+                            'AppendixEntityImage': childWidget.inputAppendixImageEdit.text()}
+            appendixNotes.append(appendixDict)
+        return data, appendixNotes
+
+
+class AppendixWidget(QtWidgets.QWidget):
+
+    def __init__(self) -> None:
+        super(AppendixWidget, self).__init__()
+        appendixWidgetLayout = QtWidgets.QGridLayout()
+        appendixLabelNotes = QtWidgets.QLabel("Entity Notes: ")
+        self.inputAppendixNotesEdit = QtWidgets.QPlainTextEdit()
+        imageAppendixLabel = QtWidgets.QLabel("Image Path: ")
+        appendixButton = QtWidgets.QPushButton("Select Image...")
+        appendixButton.clicked.connect(self.editAppendixPath)
+        self.inputAppendixImageEdit = QtWidgets.QLineEdit()
+        self.inputAppendixImageEdit.setReadOnly(True)
+        appendixWidgetLayout.addWidget(appendixLabelNotes, 0, 0, 1, 1)
+        appendixWidgetLayout.addWidget(self.inputAppendixNotesEdit, 2, 0, 4, 1)
+        appendixWidgetLayout.addWidget(imageAppendixLabel, 7, 0, 1, 1)
+        appendixWidgetLayout.addWidget(self.inputAppendixImageEdit, 9, 0, 1, 1)
+        appendixWidgetLayout.addWidget(appendixButton, 11, 0, 1, 1)
+        self.setLayout(appendixWidgetLayout)
+        self.inputAppendixNotesEdit.setFixedHeight(100)
+
+    def editAppendixPath(self) -> None:
+        selectedPath = QtWidgets.QFileDialog().getOpenFileName(parent=self, caption='Select New Icon',
+                                                               dir=str(Path.home()),
+                                                               options=QtWidgets.QFileDialog.Option.DontUseNativeDialog,
+                                                               filter="Image Files (*.png *.jpg)")[0]
+
+        if selectedPath != '':
+            self.inputAppendixImageEdit.setText(str(Path(selectedPath).absolute()))
 
 
 class MyDocTemplate(BaseDocTemplate):
@@ -32,7 +397,7 @@ class MyDocTemplate(BaseDocTemplate):
             style = flowable.style.name
             if style == 'Heading1':
                 self.notify('TOCEntry', (0, text, self.page))
-            if style == 'Heading2':
+            elif style == 'Heading2':
                 self.notify('TOCEntry', (1, text, self.page))
 
 
@@ -57,7 +422,7 @@ class ReportBuilder(canvas.Canvas):
         canvas.Canvas.save(self)
 
     def drawHeaderAndFooter(self, pageCount):
-        pageCountString = "Page %s of %s" % (self._pageNumber, pageCount)
+        pageCountString = f"Page {self._pageNumber} of {pageCount}"
         self.saveState()
         self.setStrokeColorRGB(0, 0, 0)
         self.setLineWidth(0.5)
@@ -95,7 +460,7 @@ class PDFReport:
         self.elements.append(spacer)
         self.elements.append(report_subtitle)
 
-        spacer = Spacer(100, 200)
+        spacer = Spacer(100, 425)
         self.elements.append(spacer)
 
         date = datetime.now().strftime('%Y-%m-%d %H:%m:%S')
@@ -122,8 +487,9 @@ class PDFReport:
         # path to image needed
         img = Image(summaryCanvasImage)
         img.preserveAspectRatio = True
-        img.drawHeight = 5.5 * inch
-        img.drawWidth = 6.5 * inch
+        img._restrictSize(6.5 * inch, 5.5 * inch)
+        # img.drawHeight = 5.5 * inch
+        # img.drawWidth = 6.5 * inch
         img.hAlign = 'CENTER'
 
         self.elements.append(paragraphReportSummary)
@@ -163,14 +529,16 @@ class PDFReport:
         """
 
         text = Paragraph(notes, notesParagraph)
-        if type(entityImagePath) == Drawing:
+        if isinstance(entityImagePath, Drawing):
             tbl = ReportImageAndParagraph(text, entityImagePath, side='left', xpad=10, ypad=0)
         elif entityImagePath.endswith('.png') or entityImagePath.endswith('.jpg'):
-            entityImage = Image(entityImagePath, kind='proportional')
+            entityImage = Image(entityImagePath)
             entityImage.preserveAspectRatio = True
             entityImage.drawHeight = 2 * inch
             entityImage.drawWidth = 2 * inch
             tbl = ReportImageAndParagraph(text, entityImage, side='left', xpad=10, ypad=0)
+        else:
+            raise ValueError('Invalid Image Type')
 
         links_subHeader = Paragraph("Entity Links", psSubHeaderText)
 
@@ -186,46 +554,56 @@ class PDFReport:
             outgoing_data = [
                 ['Outgoing Links'],
                 ['Resolution Name', 'Child Entity', 'Date Created', 'Notes']]
-            index = 0
-            for link in outgoingLinks:
-                linkName = link['Resolution']
-                parentNode = outgoingNames[index]
-                linkNotes = link['Notes']
-                dateCreated = link['Date Created']
-                outgoing_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
-                index += 1
-            outgoing_table = Table(data=outgoing_data, style=links_table_style, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+            for index, link in enumerate(outgoingLinks):
+                resolutionText = link['Resolution']
+                childNodeText = outgoingNames[index]
+                linkNotesText = link['Notes']
+                dateCreatedText = link['Date Created']
+                linkName = "".join([resolutionText[counter:counter + 24] + "\n"
+                                    for counter in range(0, len(resolutionText), 24)])
+                childNode = "".join([childNodeText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(childNodeText), 24)])
+                linkNotes = "".join([linkNotesText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(linkNotesText), 24)])
+                dateCreated = "".join([dateCreatedText[counter:counter + 24] + "\n"
+                                       for counter in range(0, len(dateCreatedText), 24)])
+                outgoing_data.append([linkName, childNode, dateCreated, Paragraph(linkNotes)])
+            outgoing_table = Table(data=outgoing_data, style=links_table_style, hAlign="CENTER",
+                                   colWidths=[140, 140, 140, 140])
         else:
             outgoing_data = [
                 ['No Outgoing Links']]
             outgoing_table = Table(data=outgoing_data, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
 
         # parent UID
         if incomingLinks:
             incoming_data = [
                 ['Incoming Links'],
                 ['Resolution Name', 'Parent Entity', 'Date Created', 'Notes']]
-            index = 0
-            for link in incomingLinks:
-                linkName = link['Resolution']
-                parentNode = incomingNames[index]
-                linkNotes = link['Notes']
-                dateCreated = link['Date Created']
+            for index, link in enumerate(incomingLinks):
+                resolutionText = link['Resolution']
+                parentNodeText = incomingNames[index]
+                linkNotesText = link['Notes']
+                dateCreatedText = link['Date Created']
+                linkName = "".join([resolutionText[counter:counter + 24] + "\n"
+                                    for counter in range(0, len(resolutionText), 24)])
+                parentNode = "".join([parentNodeText[counter:counter + 24] + "\n"
+                                      for counter in range(0, len(parentNodeText), 24)])
+                linkNotes = "".join([linkNotesText[counter:counter + 24] + "\n"
+                                     for counter in range(0, len(linkNotesText), 24)])
+                dateCreated = "".join([dateCreatedText[counter:counter + 24] + "\n"
+                                       for counter in range(0, len(dateCreatedText), 24)])
                 incoming_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
-                index += 1
-            incoming_table = Table(data=incoming_data, style=links_table_style, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+            incoming_table = Table(data=incoming_data, style=links_table_style, hAlign="CENTER",
+                                   colWidths=[140, 140, 140, 140])
         else:
             incoming_data = [
                 ['No Incoming Links']]
             incoming_table = Table(data=incoming_data, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
 
         tableParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_CENTER,
                                         justifyLastLine=0)
@@ -242,36 +620,16 @@ class PDFReport:
                               ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
                               ('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
         for key in list(entity):
-            if key != 'uid' and key != 'Icon' and key != 'Notes' and key != 'Date Last Edited':
-                value = entity[key]
+            if key not in avoid_parsing_fields and key != 'Notes':
+                valueText = entity[key]
+                value = "".join([valueText[counter:counter + 80] + "\n"
+                                 for counter in range(0, len(valueText), 80)])
                 entity_data.append([Paragraph(key, tableParagraph), Paragraph(value, tableParagraph)])
             elif key == 'Notes':
                 entity_notes_header = Paragraph("Entity Notes ", notesHeader)
                 entity_notes = Paragraph(entity[key], notesParagraph)
         entity_table = Table(data=entity_data, style=entity_table_style, hAlign="CENTER")
 
-        appendix_header = Paragraph(f"Appendix {appendixNumber}", notesHeader)
-        text = []
-        images = []
-        imangeNParagraph = []
-        for appendixDict in appendixDicts:
-            if appendixDict['AppendixEntityImage'] == '':
-                text.append(Paragraph(appendixDict['AppendixEntityNotes'], notesParagraph))
-            elif appendixDict['AppendixEntityNotes'] == '' and appendixDict['AppendixEntityImage'] != '':
-                img = Image(Path(appendixDict['AppendixEntityImage']), kind='proportional')
-                # img.preserveAspectRatio=True
-                img.drawHeight = 2 * inch
-                img.drawWidth = 2 * inch
-                img.hAlign = 'LEFT'
-                images.append(img)
-            elif appendixDict['AppendixEntityNotes'] != '' and appendixDict['AppendixEntityImage'] != '':
-                paragraph = appendixDict['AppendixEntityNotes']
-                img = Image(Path(appendixDict['AppendixEntityImage']), kind='proportional')
-                img.drawHeight = 2 * inch
-                img.drawWidth = 2 * inch
-                imangeNParagraph.append(
-                    ReportImageAndParagraph(Paragraph(paragraph), img, side='left', xpad=10, ypad=0))
-
         self.elements.append(entityTitle)
         spacer = Spacer(20, 20)
         self.elements.append(spacer)
@@ -289,17 +647,18 @@ class PDFReport:
         self.elements.append(outgoing_table)
 
         if not incomingLinks and not outgoingLinks:
-            print('No links to draw link pie graph for: ' + title)
+            # No links to draw link pie graph.
+            pass
         else:
             pieDraw = Drawing()
             pie = Pie()
             pie.x = 150
             pie.y = 65
-            pie.data = [int(len(incomingLinks)), int(len(outgoingLinks))]
+            pie.data = [len(incomingLinks), len(outgoingLinks)]
             pie.sideLabels = 1
-            pie.labels = ['Incoming: ' + str(len(incomingLinks)), 'Outgoing: ' + str(len(outgoingLinks))]
+            pie.labels = [f'Incoming: {len(incomingLinks)}', f'Outgoing: {len(outgoingLinks)}']
             pie.slices.strokeWidth = 1
-            if int(len(incomingLinks)) > int(len(outgoingLinks)):
+            if len(incomingLinks) > len(outgoingLinks):
                 pie.slices[0].popout = 5
             else:
                 pie.slices[1].popout = 5
@@ -314,21 +673,34 @@ class PDFReport:
         self.elements.append(entity_notes_header)
         self.elements.append(entity_notes)
         self.elements.append(spacer)
-        self.elements.append(appendix_header)
+
         spacer = Spacer(20, 20)
-        self.elements.append(spacer)
-        for elementText in text:
-            self.elements.append(elementText)
+        for appendixIndex, appendixDict in enumerate(appendixDicts):
+            appendix_header = Paragraph(f"Entity Appendix {appendixIndex}", notesHeader)
+            self.elements.append(appendix_header)
             self.elements.append(spacer)
-        for elementImage in images:
-            self.elements.append(elementImage)
-            self.elements.append(spacer)
-        for elementBoth in imangeNParagraph:
-            self.elements.append(elementBoth)
+
+            appendixImage = appendixDict['AppendixEntityImage']
+            appendixNotes = appendixDict['AppendixEntityNotes']
+            if appendixImage == '':
+                self.elements.append(Paragraph(appendixNotes, notesParagraph))
+            elif appendixNotes == '' and appendixImage != '':
+                img = Image(Path(appendixImage))
+                # img.preserveAspectRatio = True
+                img.drawHeight = 3 * inch
+                img.drawWidth = 3 * inch
+                self.elements.append(img)
+            elif appendixNotes != '' and appendixImage != '':
+                paragraph = appendixNotes
+                img = Image(Path(appendixImage))
+                img.drawHeight = 3 * inch
+                img.drawWidth = 3 * inch
+                self.elements.append(ReportImageAndParagraph(Paragraph(paragraph), img, side='left', xpad=10, ypad=0))
             self.elements.append(spacer)
 
         self.elements.append(PageBreak())
 
+    # Currently, does not really help very much and does not look good, so the graph page is not generated.
     def graphPage(self, timeLineImage):
         spacer = Spacer(10, 10)
         self.elements.append(spacer)
@@ -336,7 +708,7 @@ class PDFReport:
         ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_LEFT,
                        justifyLastLine=1)
 
-        img = Image(timeLineImage, kind='proportional')
+        img = Image(timeLineImage)
         img.drawHeight = 1.3 * inch
         img.drawWidth = 6 * inch
         img.hAlign = 'LEFT'
@@ -409,16 +781,21 @@ class PDFReport:
 
             head = f'Entity Report: {entityPrimaryField[i - 3]}'
             self.nextPagesHeader(True, head)
-            self.entityPage(title=entityPrimaryField[i - 3], userNotes=entityListData[i][0].get('EntityNotes'),
+            self.entityPage(title=entityPrimaryField[i - 3],
+                            userNotes=entityListData[i][0].get('EntityNotes'),
                             entityImagePath=imagePath,
-                            appendixDicts=entityListData[i][1], outgoingLinks=outgoingLinks[i - 3],
+                            appendixDicts=entityListData[i][1],
+                            outgoingLinks=outgoingLinks[i - 3],
                             incomingLinks=incomingLinks[i - 3],
-                            entity=entity[i - 3], incomingNames=incomingNames[i - 3], outgoingNames=outgoingNames[i - 3],
+                            entity=entity[i - 3],
+                            incomingNames=incomingNames[i - 3],
+                            outgoingNames=outgoingNames[i - 3],
                             appendixNumber=i - 3)
 
-        head = 'Graph Report'
-        self.nextPagesHeader(True, head)
-        self.graphPage(timelineImage)
+        # Graph report stuff disabled, at least for now.
+        # head = 'Graph Report'
+        # self.nextPagesHeader(True, head)
+        # self.graphPage(timelineImage)
 
         # Build
         self.doc = MyDocTemplate(path)

Core/ResolutionManager.py

@@ -1,24 +1,35 @@
 #!/usr/bin/env python3
 
-import importlib
+import contextlib
+import importlib.util
 import sys
 from os import listdir
 from pathlib import Path
+from uuid import uuid4
+from typing import Union, Any
+
+from PySide6 import QtCore, QtWidgets, QtGui
+from Core.ResourceHandler import StringPropertyInput, FilePropertyInput, SingleChoicePropertyInput, \
+    MultiChoicePropertyInput
 
 
 class ResolutionManager:
 
     # Load all resources needed.
-    def __init__(self, mainWindow, messageHandler):
-        self.messageHandler = messageHandler
+    def __init__(self, mainWindow):
         self.mainWindow = mainWindow
         self.resolutions = {}
+        # Macro dict item contents: tuple of (resolution, parameter values)
+        self.macros = {}
 
-    def loadResolutionsFromDir(self, directory: Path):
+        self.loadResolutionsFromDir(
+            Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Core" / "Resolutions" / "Core")
+
+    def loadResolutionsFromDir(self, directory: Path) -> list:
+        resolutionsLoaded = []
         exceptionsCount = 0
-        if self.resolutions.get(directory.stem) is None:
-            self.resolutions[directory.stem] = {}
         for resolution in listdir(directory):
+            resolution = str(resolution)
             try:
                 if resolution.endswith('.py'):
                     resolutionName = resolution[:-3]
@@ -34,53 +45,76 @@ class ResolutionManager:
                     originTypes = resClassInst.originTypes
                     resultTypes = resClassInst.resultTypes
                     resolutionParameters = resClassInst.parameters
-                    self.resolutions[directory.stem][resNameString] = {'name': resNameString,
-                                                                       'description': resolutionDesc,
-                                                                       'originTypes': originTypes,
-                                                                       'resultTypes': resultTypes,
-                                                                       'parameters': resolutionParameters,
-                                                                       'resolution': resClass
-                                                                       }
-                    self.messageHandler.info("Loaded Resolution: " + resNameString)
+                    with contextlib.suppress(AttributeError):
+                        resolutionCategory = resClassInst.category
+                        if not isinstance(resolutionCategory, str):
+                            resolutionCategory = "Uncategorized"
+                    if self.resolutions.get(resolutionCategory) is None:
+                        self.resolutions[resolutionCategory] = {}
+                    self.resolutions[resolutionCategory][resNameString] = {'name': resNameString,
+                                                                           'description': resolutionDesc,
+                                                                           'originTypes': originTypes,
+                                                                           'resultTypes': resultTypes,
+                                                                           'parameters': resolutionParameters,
+                                                                           'category': resolutionCategory,
+                                                                           'resolution': resClass
+                                                                           }
+                    self.mainWindow.MESSAGEHANDLER.debug(f"Loaded Resolution: {resNameString}")
+                    resolutionsLoaded.append(f'{resolutionCategory}/{resNameString}')
             except Exception as e:
-                self.messageHandler.error("Cannot load resolutions from " + str(directory) + "\n Info: " + repr(e))
+                self.mainWindow.MESSAGEHANDLER.error(
+                    f"Cannot load resolutions from {str(directory)}\n Info: {repr(e)}")
                 exceptionsCount += 1
-                if exceptionsCount > 5:
-                    self.messageHandler.critical("Failed loading too many modules to proceed.")
+                if exceptionsCount > 3:
+                    # Will not occur when loading modules with 3 or fewer resolutions, but that should be fine.
+                    self.mainWindow.MESSAGEHANDLER.critical("Failed loading too many resolutions to proceed.")
                     sys.exit(5)
+        return resolutionsLoaded
 
     def getResolutionParameters(self, resolutionCategory, resolutionNameString):
         resolutionsList = self.resolutions.get(resolutionCategory)
         if resolutionsList is not None and resolutionNameString in resolutionsList:
-            parameters = self.resolutions[resolutionCategory][resolutionNameString]['parameters']
-            return parameters
+            return dict(self.resolutions[resolutionCategory][resolutionNameString]['parameters'])
         return None
 
-    def getResolutionOriginTypes(self, resolutionNameString):
-        for category in self.resolutions:
-            if resolutionNameString in self.resolutions[category]:
-                originTypes = self.resolutions[category][resolutionNameString]['originTypes']
+    def getResolutionOriginTypes(self, resolutionCategoryNameString: str) -> Union[list, None]:
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                originTypes = self.resolutions[resolutionCategory][resolutionName]['originTypes']
                 if '*' in originTypes:
                     originTypes = self.mainWindow.RESOURCEHANDLER.getAllEntities()
                 return originTypes
         return None
 
-    def loadResolutionsFromServer(self, serverRes):
-        self.resolutions |= serverRes
+    def getResolutionDescription(self, resolutionCategoryNameString: str) -> Union[str, None]:
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                return self.resolutions[resolutionCategory][resolutionName].get('description', '')
+        return None
 
-    def removeServerResolutions(self):
-        try:
-            self.resolutions.pop("Server Resolutions")
-        except KeyError:
-            pass
+    def loadResolutionsFromServer(self, serverRes) -> None:
+        for category in serverRes:
+            if category not in self.resolutions:
+                self.resolutions[category] = {}
+            for serverResolution in serverRes[category]:
+                self.resolutions[category][serverResolution] = serverRes[category][serverResolution]
 
-    def getResolutionCategories(self):
-        result = []
-        for category in self.resolutions:
-            result.append(category)
-        return result
+    def removeServerResolutions(self) -> None:
+        for category in dict(self.resolutions):
+            for resolution in dict(self.resolutions[category]):
+                # If resolution class does not exist locally, then assume it exists on the server.
+                if self.resolutions[category][resolution]['resolution'] == '':
+                    self.resolutions[category].pop(resolution)
+        for category in dict(self.resolutions):
+            if len(self.resolutions[category]) == 0:
+                self.resolutions.pop(category)
 
-    def getResolutionsForEntityTypesByCategory(self, eTypes):
+    def getResolutionCategories(self) -> list:
+        return list(self.resolutions)
+
+    def getResolutionsForEntityTypesByCategory(self, eTypes) -> dict:
         """
         Gets a set of entity types, and returns a dictionary with all the resolutions that can take all
         included types as input.
@@ -95,29 +129,575 @@ class ResolutionManager:
                     result[category].append(resolution)
         return result
 
-    def getResolutionsInCategory(self, category):
-        if category in self.resolutions:
-            return list(self.resolutions[category])
-        return []
+    def getResolutionsInCategory(self, category) -> list:
+        return list(self.resolutions[category]) if category in self.resolutions else []
 
-    def getAllResolutions(self):
+    def getAllResolutions(self) -> list:
         categories = self.getResolutionCategories()
         result = []
         for category in categories:
             result += self.getResolutionsInCategory(category)
         return result
 
-    def executeResolution(self, resolutionName: str, resolutionEntitiesInput: list, parameters: dict,
+    def executeResolution(self, resolutionCategoryNameString: str, resolutionEntitiesInput: list, parameters: dict,
                           resolutionUID: str):
+        resolutionCategory, resolutionName = resolutionCategoryNameString.split('/', 1)
+        with contextlib.suppress(TypeError):
+            if resolutionName in self.resolutions.get(resolutionCategory):
+                if self.resolutions[resolutionCategory][resolutionName].get('resolution') == '':
+                    # If resolution class does not exist locally, then assume it exists on the server.
+                    self.mainWindow.FCOM.runRemoteResolution(
+                        resolutionCategoryNameString, resolutionEntitiesInput, parameters, resolutionUID)
+                    # Returning a bool, so we know that the resolution is running on the server.
+                    return True
+                resolutionClass = self.resolutions[resolutionCategory][resolutionName]['resolution']()
+                return resolutionClass.resolution(resolutionEntitiesInput, parameters)
+        return None
+
+    def createMacro(self, resolutionList: list) -> str:
+        macroUID = str(uuid4())
+        self.macros[macroUID] = resolutionList
+
+        return macroUID
+
+    def renameMacro(self, oldName: str, newName: str) -> bool:
+        if oldName != newName:
+            with self.mainWindow.macrosLock:
+                if newName in self.macros:
+                    self.mainWindow.MESSAGEHANDLER.warning('The specified name already exists. '
+                                                           'Macro names must be unique.',
+                                                           popUp=True)
+                    return False
+                if oldName not in self.macros:
+                    self.mainWindow.MESSAGEHANDLER.error('Attempting to rename a nonexistent macro.', popUp=True)
+                    return False
+                oldMacro = self.macros.pop(oldName)
+                self.macros[newName] = oldMacro
+        return True
+
+    def deleteMacro(self, macroUID: str) -> bool:
+        # We don't have to worry about running macros, because the details of the macro are saved in memory.
+        # We do however want to get the thread lock because of potential race conditions.
+        try:
+            with self.mainWindow.macrosLock:
+                self.macros.pop(macroUID)
+            return True
+        except KeyError:
+            return False
+
+    def loadMacros(self) -> None:
+        # Load AFTER we load resolutions.
+        self.macros = self.mainWindow.SETTINGS.value("Program/Macros", {})
+
+    def save(self) -> None:
+        self.mainWindow.SETTINGS.setGlobalValue("Program/Macros", self.macros)
+
+
+class ResolutionParametersSelector(QtWidgets.QDialog):
+
+    def __init__(self, mainWindowObject, resolutionName, properties: dict, includeEntitySelector: list = None,
+                 originTypes: list = None, resolutionDescription: str = None,
+                 windowTitle: str = None) -> None:
+        super(ResolutionParametersSelector, self).__init__()
+
+        self.setModal(True)
+        if windowTitle is None:
+            windowTitle = f'Resolution Parameter Selector: {resolutionName}'
+        self.setWindowTitle(windowTitle)
+        self.parametersList = []
+        # Have two separate dicts for readability.
+        self.chosenParameters = {}
+        self.properties = properties
+        self.mainWindowObject = mainWindowObject
+        self.resolutionName = resolutionName
+
+        dialogLayout = QtWidgets.QGridLayout()
+        self.setLayout(dialogLayout)
+        self.childWidget = QtWidgets.QTabWidget()
+        dialogLayout.addWidget(self.childWidget, 0, 0, 4, 2)
+        dialogLayout.setRowStretch(0, 1)
+        dialogLayout.setColumnStretch(0, 1)
+
+        if includeEntitySelector is not None and originTypes is not None:
+            entitySelectTab = QtWidgets.QWidget()
+            entitySelectTab.setLayout(QtWidgets.QVBoxLayout())
+            labelText = ""
+            if resolutionDescription is not None:
+                labelText += resolutionDescription + "\n\n"
+            labelText += 'Select the entities to use for this resolution.\nAccepted Origin Types: ' + \
+                         ', '.join(originTypes)
+            entitySelectTabLabel = QtWidgets.QLabel(labelText)
+            entitySelectTabLabel.setWordWrap(True)
+            entitySelectTabLabel.setMaximumWidth(600)
+
+            entitySelectTabLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+            entitySelectTab.layout().addWidget(entitySelectTabLabel)
+
+            self.entitySelector = QtWidgets.QListWidget()
+            self.entitySelector.setSortingEnabled(True)
+            self.entitySelector.addItems(includeEntitySelector)
+
+            self.entitySelector.setSelectionMode(self.entitySelector.SelectionMode.MultiSelection)
+            entitySelectTab.layout().addWidget(self.entitySelector)
+
+            self.childWidget.addTab(entitySelectTab, 'Entities')
+
+        for key in properties:
+            propertyWidget = QtWidgets.QWidget()
+            propertyKeyLayout = QtWidgets.QVBoxLayout()
+            propertyWidget.setLayout(propertyKeyLayout)
+
+            propertyLabel = QtWidgets.QLabel(properties[key].get('description'))
+            propertyLabel.setWordWrap(True)
+            propertyLabel.setMaximumWidth(600)
+
+            propertyLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+            propertyKeyLayout.addWidget(propertyLabel)
+
+            propertyType = properties[key].get('type')
+            propertyValue = properties[key].get('value')
+            propertyDefaultValue = properties[key].get('default')
+
+            if propertyType == 'String':
+                propertyInputField = StringPropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'File':
+                propertyInputField = FilePropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'SingleChoice':
+                propertyInputField = SingleChoicePropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'MultiChoice':
+                propertyInputField = MultiChoicePropertyInput(propertyValue, propertyDefaultValue)
+            else:
+                # If value has invalid type, skip to the next property.
+                propertyInputField = None
+
+            if propertyInputField is not None:
+                propertyKeyLayout.addWidget(propertyInputField)
+
+            rememberChoiceCheckbox = QtWidgets.QCheckBox('Remember Choice')
+            rememberChoiceCheckbox.setChecked(False)
+            propertyKeyLayout.addWidget(rememberChoiceCheckbox)
+            propertyKeyLayout.setStretch(1, 1)
+
+            self.childWidget.addTab(propertyWidget, key)
+            self.parametersList.append((key, propertyInputField, rememberChoiceCheckbox))
+
+        nextButton = QtWidgets.QPushButton('Next')
+        nextButton.clicked.connect(self.nextTab)
+        previousButton = QtWidgets.QPushButton('Previous')
+        previousButton.clicked.connect(self.previousTab)
+        acceptButton = QtWidgets.QPushButton('Accept')
+        acceptButton.setAutoDefault(True)
+        acceptButton.setDefault(True)
+        acceptButton.clicked.connect(self.accept)
+        cancelButton = QtWidgets.QPushButton('Cancel')
+        cancelButton.clicked.connect(self.reject)
+
+        dialogLayout.addWidget(previousButton, 4, 0, 1, 1)
+        dialogLayout.addWidget(nextButton, 4, 1, 1, 1)
+        dialogLayout.addWidget(cancelButton, 5, 0, 1, 1)
+        dialogLayout.addWidget(acceptButton, 5, 1, 1, 1)
+
+    def nextTab(self):
+        currentIndex = self.childWidget.currentIndex()
+        if currentIndex < self.childWidget.count():
+            self.childWidget.setCurrentIndex(currentIndex + 1)
+
+    def previousTab(self):
+        currentIndex = self.childWidget.currentIndex()
+        if currentIndex > 0:
+            self.childWidget.setCurrentIndex(currentIndex - 1)
+
+    def accept(self) -> None:
+        savedParameters = {}
+        for resolutionParameterName, resolutionParameterInput, resolutionParameterRemember in self.parametersList:
+            value = resolutionParameterInput.getValue()
+            if value == '':
+                msgBox = QtWidgets.QMessageBox()
+                msgBox.setModal(True)
+                QtWidgets.QMessageBox.warning(msgBox,
+                                              "Not all parameters were filled in",
+                                              "Some of the required parameters for the resolution have been left blank."
+                                              " Please fill them in before proceeding.")
+                return
+            self.chosenParameters[resolutionParameterName] = value
+
+            if resolutionParameterRemember.isChecked():
+                savedParameters[resolutionParameterName] = value
+
+        # Only save parameters after we verify that everything is filled in properly.
+        for savedParameter in savedParameters:
+            if self.properties[savedParameter].get('global') is True:
+                self.mainWindowObject.SETTINGS.setGlobalValue(
+                    f'Resolutions/Global/Parameters/{savedParameter}',
+                    savedParameters[savedParameter],
+                )
+            else:
+                self.mainWindowObject.SETTINGS.setGlobalValue(
+                    f'Resolutions/{self.resolutionName}/{savedParameter}',
+                    savedParameters[savedParameter],
+                )
+
+        super(ResolutionParametersSelector, self).accept()
+
+
+class ResolutionSearchResultsList(QtWidgets.QListWidget):
+
+    def __init__(self, mainWindowObject):
+        super(ResolutionSearchResultsList, self).__init__()
+        self.mainWindow = mainWindowObject
+        self.setSortingEnabled(True)
+
+    def mouseDoubleClickEvent(self, event: QtGui.QMouseEvent) -> None:
+        super(ResolutionSearchResultsList, self).mouseDoubleClickEvent(event)
+        resItem = self.itemAt(event.pos())
+        if resItem is None or '/' not in resItem.text():
+            return
+        self.mainWindow.centralWidget().tabbedPane.getCurrentScene().clearSelection()
+        self.mainWindow.runResolution(resItem.text())
+
+
+class FindResolutionDialog(QtWidgets.QDialog):
+
+    def __init__(self, parent, entityList: list, resolutionDict: dict):
+        super(FindResolutionDialog, self).__init__()
+        self.entities = entityList
+        self.resolutions = resolutionDict
+        self.setModal(True)
+        self.setWindowTitle('Find Resolutions')
+
+        dialogLayout = QtWidgets.QGridLayout()
+        self.setLayout(dialogLayout)
+
+        descriptionLabel = QtWidgets.QLabel("Find Resolutions based on their parameters.")
+        descriptionLabel.setWordWrap(True)
+        dialogLayout.addWidget(descriptionLabel, 0, 0, 1, 2)
+
+        originLabel = QtWidgets.QLabel("Origin Entity:")
+        self.originDropDown = QtWidgets.QComboBox()
+        self.originDropDown.addItem('Any')
+        self.originDropDown.addItems(entityList)
+        self.originDropDown.addItem('*')
+        dialogLayout.addWidget(originLabel, 1, 0, 1, 1)
+        dialogLayout.addWidget(self.originDropDown, 1, 1, 1, 1)
+
+        targetLabel = QtWidgets.QLabel("Target Entity:")
+        self.targetDropDown = QtWidgets.QComboBox()
+        self.targetDropDown.addItem('Any')
+        self.targetDropDown.addItems(entityList)
+        self.targetDropDown.addItem('*')
+        dialogLayout.addWidget(targetLabel, 2, 0, 1, 1)
+        dialogLayout.addWidget(self.targetDropDown, 2, 1, 1, 1)
+
+        keywordsLabel = QtWidgets.QLabel("Keywords:")
+        self.keywordsWidget = QtWidgets.QLineEdit()
+        self.keywordsWidget.setToolTip("Add keywords separated by spaces.\nKeywords are checked against the "
+                                       "resolutions' titles and descriptions.")
+        dialogLayout.addWidget(keywordsLabel, 3, 0, 1, 2)
+        dialogLayout.addWidget(self.keywordsWidget, 4, 0, 1, 2)
+
+        resultsLabel = QtWidgets.QLabel("Matches:")
+        self.resultsWidget = ResolutionSearchResultsList(parent)
+        self.resultsWidget.addItem('Click "Search" to display results')
+        dialogLayout.addWidget(resultsLabel, 5, 0, 1, 2)
+        dialogLayout.addWidget(self.resultsWidget, 6, 0, 2, 2)
+
+        self.searchButton = QtWidgets.QPushButton("Search")
+        self.searchButton.clicked.connect(self.search)
+        self.closeButton = QtWidgets.QPushButton("Close")
+        self.closeButton.clicked.connect(self.accept)
+        dialogLayout.addWidget(self.closeButton, 8, 0, 1, 1)
+        dialogLayout.addWidget(self.searchButton, 8, 1, 1, 1)
+
+    def search(self):
+        self.resultsWidget.clear()
+        target = self.targetDropDown.currentText()
+        validResolutions = []
         for category in self.resolutions:
             for resolution in self.resolutions[category]:
-                if self.resolutions[category][resolution]['name'] == resolutionName:
-                    if category == "Server Resolutions":
-                        self.mainWindow.executeRemoteResolution(resolutionName, resolutionEntitiesInput, parameters,
-                                                                resolutionUID)
-                        # Returning a bool so we know that the resolution is running on the server.
-                        return True
+                if target == 'Any':
+                    validResolutions.append(f'{category}/{resolution}')
+
+                elif target in self.resolutions[category][resolution]['originTypes']:
+                    validResolutions.append(f'{category}/{resolution}')
+        origin = self.originDropDown.currentText()
+        if origin != 'Any':
+            for category in self.resolutions:
+                for resolution in self.resolutions[category]:
+                    if origin not in self.resolutions[category][resolution]['originTypes']:
+                        with contextlib.suppress(KeyError):
+                            validResolutions.remove(f'{str(category)}/{str(resolution)}')
+        # Try to see if any of the keywords are a substring of the name or description of any resolution.
+        keywordFilter = self.keywordsWidget.text().strip()
+        if keywordFilter != '':
+            wordsToFind = keywordFilter.split(' ')
+            for category in self.resolutions:
+                for resolution in self.resolutions[category]:
+                    titleText = self.resolutions[category][resolution]['name']
+                    descriptionText = self.resolutions[category][resolution]['description']
+                    for keyword in wordsToFind:
+                        if keyword not in titleText and keyword not in descriptionText:
+                            with contextlib.suppress(KeyError):
+                                validResolutions.remove(f'{str(category)}/{str(resolution)}')
+        for result in validResolutions:
+            self.resultsWidget.addItem(result)
+
+
+class ResolutionExecutorThread(QtCore.QThread):
+    sig = QtCore.Signal(str, list, str)
+    sigStr = QtCore.Signal(str, str, str)
+    sigError = QtCore.Signal(str)
+
+    def __init__(self, resolution: str, resolutionArgument: list, resolutionParameters: dict,
+                 mainWindowObject, uid: str):
+        super().__init__()
+        self.resolution = resolution
+        self.resolutionArgument = resolutionArgument
+        self.resolutionParameters = resolutionParameters
+        self.mainWindow = mainWindowObject
+        self.return_results = True
+        self.uid = uid
+        self.done = False
+
+    def run(self) -> None:
+        try:
+            ret = self.mainWindow.RESOLUTIONMANAGER.executeResolution(self.resolution,
+                                                                      self.resolutionArgument,
+                                                                      self.resolutionParameters,
+                                                                      self.uid)
+            if ret is None:
+                self.sigError.emit(f'Resolution {self.resolution} failed during run.')
+            elif isinstance(ret, bool):
+                # Resolution is running on the server, we do not have results right now.
+                ret = None
+        except Exception as e:
+            self.sigError.emit(f'Resolution {self.resolution} failed during run: {str(e)}')
+            ret = None
+
+        # If the resolution is ran on the server or there is a problem, don't emit signal.
+        if ret is not None and self.return_results:
+            if isinstance(ret, str):
+                self.sigStr.emit(self.resolution, ret, self.uid)
+            else:
+                self.sig.emit(self.resolution, ret, self.uid)
+            self.done = True
+
+
+class MacroDialog(QtWidgets.QDialog):
+
+    def __init__(self, mainWindowObject):
+        super(MacroDialog, self).__init__()
+        self.mainWindowObject = mainWindowObject
+        self.setModal(True)
+
+        self.resolutionList = []
+        for category in mainWindowObject.RESOLUTIONMANAGER.getResolutionCategories():
+            self.resolutionList.extend(
+                f'{category}/{resolution}'
+                for resolution in mainWindowObject.RESOLUTIONMANAGER.getResolutionsInCategory(category)
+            )
+        self.resolutionList.sort()
+
+        layout = QtWidgets.QVBoxLayout()
+        self.setLayout(layout)
+
+        macroLabel = QtWidgets.QLabel("This is a list of all currently configured Macros.\n"
+                                      "Click on a Macro to view the Resolutions included in it.")
+        macroLabel.setWordWrap(True)
+        self.macroTree = MacroTree(self, mainWindowObject)
+        self.macroTree.setSelectionMode(self.macroTree.SelectionMode.ExtendedSelection)
+        self.macroTree.setSelectionBehavior(self.macroTree.SelectionBehavior.SelectRows)
+        self.macroTree.setHeaderLabels(['Macro UID', 'Delete'])
+        self.macroTree.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectionBehavior.SelectRows)
+        self.macroTree.setSortingEnabled(False)
+        # Stretch the first column, since it contains the primary field.
+        self.macroTree.header().setStretchLastSection(False)
+        self.macroTree.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeMode.Stretch)
+
+        buttonsWidget = QtWidgets.QWidget()
+        buttonsWidgetLayout = QtWidgets.QHBoxLayout()
+        buttonsWidget.setLayout(buttonsWidgetLayout)
+        closeButton = QtWidgets.QPushButton('Close')
+        closeButton.clicked.connect(self.reject)
+        addMacroButton = QtWidgets.QPushButton('Create New Macro')
+        addMacroButton.clicked.connect(self.createMacro)
+        runSelectedButton = QtWidgets.QPushButton('Run Selected Macros')
+        runSelectedButton.clicked.connect(self.accept)
+        buttonsWidgetLayout.addWidget(closeButton)
+        buttonsWidgetLayout.addWidget(addMacroButton)
+        buttonsWidgetLayout.addWidget(runSelectedButton)
+
+        layout.addWidget(macroLabel)
+        layout.addWidget(self.macroTree)
+        layout.addWidget(buttonsWidget)
+        self.updateMacroTree()
+        self.setBaseSize(1000, 1000)
+
+    def updateMacroTree(self) -> None:
+        self.macroTree.clear()
+        with self.mainWindowObject.macrosLock:
+            allMacros = self.mainWindowObject.RESOLUTIONMANAGER.macros
+
+            for macro in allMacros:
+                newMacro = MacroTreeItem(macro, allMacros[macro])
+                self.macroTree.addTopLevelItem(newMacro)
+                self.macroTree.setItemWidget(newMacro, 1, newMacro.deleteButton)
+
+    def createMacro(self) -> None:
+        createMacroDialog = MacroCreatorDialog(self.resolutionList)
+        if createMacroDialog.exec():
+            macroResolutionsList = []
+            numberOfResolutionsSelected = createMacroDialog.createList.count()
+            for itemIndex in range(numberOfResolutionsSelected):
+                itemText = createMacroDialog.createList.item(itemIndex).text()
+                resolutionCategory, resolutionName = itemText.split('/', 1)
+                rParameters = self.mainWindowObject.RESOLUTIONMANAGER.getResolutionParameters(resolutionCategory,
+                                                                                              resolutionName)
+                if rParameters is None:
+                    message = f'Resolution parameters not found for resolution: {resolutionName}'
+                    self.mainWindowObject.MESSAGEHANDLER.error(message, popUp=True, exc_info=False)
+                    self.mainWindowObject.setStatus(f'{message}, Macro creation aborted.')
+                    return
+
+                resolutionParameterValues = self.mainWindowObject.popParameterValuesAndReturnSpecified(resolutionName,
+                                                                                                       rParameters)
+
+                if rParameters:
+                    parameterSelector = ResolutionParametersSelector(
+                        self.mainWindowObject, resolutionName, rParameters,
+                        windowTitle=f'[{str(itemIndex + 1)}/{str(numberOfResolutionsSelected)}] Select Parameter '
+                                    f'values for Resolution: {resolutionName}')
+                    if parameterSelector.exec():
+                        resolutionParameterValues.update(parameterSelector.chosenParameters)
+                    else:
+                        self.mainWindowObject.MESSAGEHANDLER.info('Macro creation aborted.')
+                        self.mainWindowObject.setStatus('Macro creation aborted.')
+                        return
+
+                macroResolutionsList.append((itemText, resolutionParameterValues))
+            self.mainWindowObject.RESOLUTIONMANAGER.createMacro(macroResolutionsList)
+            self.updateMacroTree()
+            self.mainWindowObject.setStatus('New Macro Created.')
+            self.mainWindowObject.MESSAGEHANDLER.info('New Macro Created.')
+
+    def accept(self) -> None:
+        super(MacroDialog, self).accept()
+
+
+class MacroTree(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindowObject):
+        super(MacroTree, self).__init__(parent=parent)
+        self.mainWindowObject = mainWindowObject
+
+    def deleteMacro(self, treeEntry: QtWidgets.QTreeWidgetItem):
+        index = self.indexOfTopLevelItem(treeEntry)
+        uid = treeEntry.text(0)
+        self.mainWindowObject.RESOLUTIONMANAGER.deleteMacro(uid)
+        self.takeTopLevelItem(index)
+
+
+class MacroTreeItem(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, uid: str, resolutionList: list):
+        super(MacroTreeItem, self).__init__()
+        self.setText(0, uid)
+        self.uid = uid
+        self.setFlags(QtCore.Qt.ItemFlag.ItemIsEditable | self.flags())
+
+        self.deleteButton = QtWidgets.QPushButton('X')
+        self.deleteButton.clicked.connect(self.removeSelf)
+
+        for resolution in resolutionList:
+            resolutionItem = QtWidgets.QTreeWidgetItem()
+            resolutionItem.setText(0, f'Resolution: {resolution[0]}')
+            self.addChild(resolutionItem)
+
+    def setData(self, column: int, role: int, value: Any):
+        if self.treeWidget() is None:
+            # Happens during initialization
+            super().setData(column, role, value)
+
+        elif self.treeWidget().mainWindowObject.RESOLUTIONMANAGER.renameMacro(self.uid, value):
+            super().setData(column, role, value)
+            self.uid = value
+
+    def removeSelf(self):
+        self.treeWidget().deleteMacro(self)
+
+
+class MacroCreatorDialog(QtWidgets.QDialog):
+
+    def __init__(self, resolutionsWithCategoriesList: list):
+        super(MacroCreatorDialog, self).__init__()
+        self.setWindowTitle('Create new Macro')
+
+        self.viewList = QtWidgets.QListWidget()
+        for resolution in resolutionsWithCategoriesList:
+            viewItem = QtWidgets.QListWidgetItem(resolution)
+            self.viewList.addItem(viewItem)
+        self.viewList.sortItems()
+        self.viewList.setSelectionMode(self.viewList.SelectionMode.ExtendedSelection)
+
+        buttonsAddRemoveWidget = QtWidgets.QWidget()
+        buttonsAddRemoveWidgetLayout = QtWidgets.QVBoxLayout()
+        buttonAdd = QtWidgets.QPushButton('>')
+        buttonAdd.clicked.connect(self.addSelectedToMacro)
+        buttonRemove = QtWidgets.QPushButton('<')
+        buttonRemove.clicked.connect(self.removeSelectedFromMacro)
+        buttonsAddRemoveWidget.setLayout(buttonsAddRemoveWidgetLayout)
+        buttonsAddRemoveWidgetLayout.addWidget(buttonAdd)
+        buttonsAddRemoveWidgetLayout.addWidget(buttonRemove)
+        self.createList = QtWidgets.QListWidget()
+        buttonsRearrangeWidget = QtWidgets.QWidget()
+        buttonsRearrangeWidgetLayout = QtWidgets.QVBoxLayout()
+        buttonsRearrangeWidget.setLayout(buttonsRearrangeWidgetLayout)
+        buttonMoveUp = QtWidgets.QPushButton('^')
+        buttonMoveUp.clicked.connect(self.shiftSelectedUp)
+        buttonMoveDown = QtWidgets.QPushButton('v')
+        buttonMoveDown.clicked.connect(self.shiftSelectedDown)
+        buttonsRearrangeWidgetLayout.addWidget(buttonMoveUp)
+        buttonsRearrangeWidgetLayout.addWidget(buttonMoveDown)
+        allResolutionsLabel = QtWidgets.QLabel('All Resolutions')
+        allResolutionsLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        selectedResolutionsLabel = QtWidgets.QLabel('Selected Resolutions')
+        selectedResolutionsLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        confirmButton = QtWidgets.QPushButton('Confirm')
+        confirmButton.clicked.connect(self.accept)
+        cancelButton = QtWidgets.QPushButton('Cancel')
+        cancelButton.clicked.connect(self.reject)
+
+        macroCreateLayout = QtWidgets.QGridLayout()
+        self.setLayout(macroCreateLayout)
+        macroCreateLayout.addWidget(self.viewList, 1, 0, 1, 5)
+        macroCreateLayout.addWidget(buttonsAddRemoveWidget, 1, 5, 1, 1)
+        macroCreateLayout.addWidget(self.createList, 1, 6, 1, 5)
+        macroCreateLayout.addWidget(buttonsRearrangeWidget, 1, 11, 1, 1)
+        macroCreateLayout.addWidget(allResolutionsLabel, 0, 0, 1, 5)
+        macroCreateLayout.addWidget(selectedResolutionsLabel, 0, 6, 1, 5)
+        macroCreateLayout.addWidget(cancelButton, 2, 1, 1, 3)
+        macroCreateLayout.addWidget(confirmButton, 2, 7, 1, 3)
+        self.setBaseSize(1000, 700)
+
+    def addSelectedToMacro(self):
+        for selectedItem in self.viewList.selectedItems():
+            self.createList.addItem(QtWidgets.QListWidgetItem(selectedItem.text()))
+
+    def removeSelectedFromMacro(self):
+        for selectedItem in self.createList.selectedItems():
+            self.createList.takeItem(self.createList.row(selectedItem))
+
+    def shiftSelectedUp(self):
+        with contextlib.suppress(IndexError):
+            selectedItemIndex = self.createList.row(self.createList.selectedItems()[0])
+            if selectedItemIndex != 0:
+                currentItem = self.createList.takeItem(selectedItemIndex)
+                self.createList.insertItem(selectedItemIndex - 1, currentItem)
+                currentItem.setSelected(True)
+
+    def shiftSelectedDown(self):
+        with contextlib.suppress(IndexError):
+            selectedItemIndex = self.createList.row(self.createList.selectedItems()[0])
+            currentItem = self.createList.takeItem(selectedItemIndex)
+            self.createList.insertItem(selectedItemIndex + 1, currentItem)
+            currentItem.setSelected(True)
 
-                    resolutionClass = self.resolutions[category][resolution]['resolution']()
-                    result = resolutionClass.resolution(resolutionEntitiesInput, parameters)
-                    return result

Core/Resolutions/Core/ASNToCIDR.py

@@ -1,15 +1,16 @@
 class ASNToCIDR:
     # A string that is treated as the name of this resolution.
     name = "Get CIDR from ASN"
+    category = "Network Infrastructure"
 
     # A string that describes this resolution.
-    description = "ASN to CIDR"
+    description = "Get network information for an Autonomous System Number."
 
     # A set of entities that this resolution can be ran on.
     originTypes = {'Autonomous System'}
 
     # A set of entities that could be the result of this resolution.
-    resultTypes = {'Network'}
+    resultTypes = {'Network', 'Company', 'Organization', 'Phrase'}
 
     # A dictionary of properties for this resolution. The key is the property name,
     # the value is the property attributes. The type of input expected from the user is determined by the
@@ -24,11 +25,14 @@ class ASNToCIDR:
 
         for entity in entityJsonList:
             if entity['Entity Type'] == 'Autonomous System':
-                ipWithPrefix = entity[list(entity)[2]]
-                uid = entity['uid']
-                split_string = ipWithPrefix.split("/", 1)
-                ipWithOutPrefix = split_string[0]
-                ASN = entity["AS Number"]
+                try:
+                    ipWithPrefix = entity['ASN Cidr']
+                    uid = entity['uid']
+                    split_string = ipWithPrefix.split("/", 1)
+                    ipWithOutPrefix = split_string[0]
+                    ASN = entity["AS Number"]
+                except Exception:
+                    continue
                 try:
                     net = Net(ipWithOutPrefix)
                 except Exception:
@@ -42,18 +46,13 @@ class ASNToCIDR:
                     cidrWithOutPrefix = split_string[0]
                     prefix = split_string[1]
                     index_of_child = len(returnResult)
-                    returnResult.append([{'IP Address': cidrWithOutPrefix,
-                                          'Range': prefix,
-                                          'Entity Type': 'Network'},
-                                         {uid: {'Resolution': 'ASN to CIDR', 'Notes': ''}}])
-                    returnResult.append(
-                        [{'Phrase': network['description'], 'Entity Type': 'Phrase'},
-                         {index_of_child: {'Resolution': 'CIDR Description', 'Notes': ''}}])
-                    returnResult.append(
-                        [{'Organization Name': network['source'], 'Entity Type': 'Organization'},
-                         {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}])
-                    returnResult.append(
-                        [{'Company Name': network['maintainer'], 'Entity Type': 'Company'},
-                         {index_of_child: {'Resolution': 'Company Name', 'Notes': ''}}])
+                    returnResult.extend(([{'IP Address': cidrWithOutPrefix, 'Range': prefix, 'Entity Type': 'Network'},
+                                          {uid: {'Resolution': 'ASN to CIDR', 'Notes': ''}}],
+                                         [{'Phrase': network['description'], 'Entity Type': 'Phrase'},
+                                          {index_of_child: {'Resolution': 'CIDR Description', 'Notes': ''}}],
+                                         [{'Organization Name': network['source'], 'Entity Type': 'Organization'},
+                                          {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}],
+                                         [{'Company Name': network['maintainer'], 'Entity Type': 'Company'},
+                                          {index_of_child: {'Resolution': 'Company Name', 'Notes': ''}}]))
 
         return returnResult

Core/Resolutions/Core/AffiliateCodesExtractor.py

@@ -13,8 +13,10 @@ class AffiliateCodesExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Affiliate Codes"
 
+    category = "Website Tracking"
+
     # A string that describes this resolution.
-    description = "Returns Nodes of facebook and amazon affiliate codes for websites"
+    description = "Returns Facebook and Amazon affiliate codes found in websites."
 
     originTypes = {'Domain', 'Website'}
 
@@ -40,14 +42,16 @@ class AffiliateCodesExtractor:
                                            'value': {'Yes', 'No'}}}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import urllib
         import tldextract
         import re
 
+        playwrightPath = Path(parameters['Playwright Firefox'])
         returnResults = []
-        visitExternal = True if parameters['Visit External Links'] == 'Yes' else False
+        visitExternal = parameters['Visit External Links'] == 'Yes'
 
         # Numbers less than zero are the same as zero, but we should try to prevent overflows.
         try:
@@ -121,6 +125,8 @@ class AffiliateCodesExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
                 return
 
@@ -129,56 +135,58 @@ class AffiliateCodesExtractor:
             linksInLinkHref = soupContents.find_all('link')
             for tag in linksInLinkHref:
                 newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractCodes(currentUID, newLink, newDepth)
+                if newLink is not None and newLink.startswith('http'):
+                    newLink = newLink.split('#')[0]
+                    newDepth = depth - 1
+                    if domain in newLink and newLink not in exploredDepth and newDepth > 0:
+                        exploredDepth.add(newLink)
+                        extractCodes(currentUID, newLink, newDepth)
 
             linksInAHref = soupContents.find_all('a')
             for tag in linksInAHref:
                 newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink:
-                            redirLinks = redirectRegex.findall(newLink)
-                            if 'redirect' in newLink and len(redirLinks) > 0:
-                                newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
-                                if newLink not in exploredForeign:
-                                    exploredForeign.add(newLink)
-                                    if visitExternal:
-                                        for _ in range(3):
-                                            try:
-                                                page.goto(newLink, wait_until="networkidle", timeout=10000)
-                                                GetAffiliateCodes(currentUID, page.url)
-                                                break
-                                            except TimeoutError:
-                                                pass
-                                else:
-                                    GetAffiliateCodes(currentUID, newLink)
-                            else:
-                                if newLink not in exploredDepth and newDepth > 0:
-                                    exploredDepth.add(newLink)
-                                    extractCodes(currentUID, newLink, newDepth)
-                        elif newLink not in exploredForeign:
-                            exploredForeign.add(newLink)
-                            if visitExternal:
-                                for _ in range(3):
-                                    try:
-                                        page.goto(newLink, wait_until="networkidle", timeout=10000)
-                                        GetAffiliateCodes(currentUID, page.url)
-                                        break
-                                    except TimeoutError:
-                                        pass
+                if newLink is not None and newLink.startswith('http'):
+                    newLink = newLink.split('#')[0]
+                    newDepth = depth - 1
+                    if domain in newLink:
+                        redirLinks = redirectRegex.findall(newLink)
+                        if 'redirect' in newLink and len(redirLinks) > 0:
+                            newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
+                            if newLink not in exploredForeign:
+                                exploredForeign.add(newLink)
+                                if visitExternal:
+                                    for _ in range(3):
+                                        try:
+                                            page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                            GetAffiliateCodes(currentUID, page.url)
+                                            break
+                                        except TimeoutError:
+                                            pass
+                                        except Error:
+                                            break
                             else:
                                 GetAffiliateCodes(currentUID, newLink)
+                        else:
+                            if newLink not in exploredDepth and newDepth > 0:
+                                exploredDepth.add(newLink)
+                                extractCodes(currentUID, newLink, newDepth)
+                    elif newLink not in exploredForeign:
+                        exploredForeign.add(newLink)
+                        if visitExternal:
+                            for _ in range(3):
+                                try:
+                                    page.goto(newLink, wait_until="networkidle", timeout=10000)
+                                    GetAffiliateCodes(currentUID, page.url)
+                                    break
+                                except TimeoutError:
+                                    pass
+                                except Error:
+                                    break
+                        else:
+                            GetAffiliateCodes(currentUID, newLink)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
@@ -189,7 +197,7 @@ class AffiliateCodesExtractor:
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
+                    url = f'http://{url}'
                 domain = tldextract.extract(url).fqdn
                 extractCodes(uid, url, maxDepth)
             browser.close()

Core/Resolutions/Core/CertificateInfo.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+
+
+class CertificateInfo:
+    name = "Analyze Website Certificate"
+    category = "Network Infrastructure"
+    description = "Get certificate information from the certificate of a certain website."
+    originTypes = {'Website'}
+    resultTypes = {'Phrase', 'Website', 'Domain', 'Country', 'Organization', 'Address', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        import ssl
+        import socket
+
+        returnResults = []
+        sslContext = ssl.create_default_context()
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            websiteURL = entity['URL']
+            if websiteURL.startswith('https'):
+                addressWithoutHttps = websiteURL[8:].split('/')[0]
+            elif websiteURL.startswith('http'):
+                # Just in case an entity was created for the http version of a site,
+                #   but there is actually a https version of the site.
+                addressWithoutHttps = websiteURL[7:].split('/')[0]
+            else:
+                continue
+
+            with sslContext.wrap_socket(socket.socket(), server_hostname=addressWithoutHttps) as s:
+                try:
+                    s.connect((addressWithoutHttps, 443))
+                    websiteCertificate = s.getpeercert()
+                except socket.error:
+                    # If there's an error connecting, move on.
+                    continue
+
+            # Subject Details
+            streetAddr = None
+            locality = None
+            postalCode = None
+            subjectCountry = None
+            subjectSerial = None
+            subjectName = None
+            tentativeSubjectResults = []
+            for subjectAttributeOuter in websiteCertificate['subject']:
+                for subjectAttributeInner in subjectAttributeOuter:
+                    subjectAttributeInnerKey = subjectAttributeInner[0]
+                    subjectAttributeInnerValue = subjectAttributeInner[1]
+                    if subjectAttributeInnerKey == 'commonName':
+                        returnResults.append([{'Domain Name': subjectAttributeInnerValue,
+                                               'Entity Type': 'Domain'},
+                                              {uid: {'Resolution': 'Certificate Subject Common Name',
+                                                     'Notes': ''}}])
+                    elif subjectAttributeInnerKey == 'countryName':
+                        subjectCountry = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'localityName':
+                        locality = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'organizationName':
+                        subjectName = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'postalCode':
+                        postalCode = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'serialNumber':
+                        subjectSerial = subjectAttributeInnerValue
+                    elif subjectAttributeInnerKey == 'streetAddress':
+                        streetAddr = subjectAttributeInnerValue
+            subjectIndex = None
+            if subjectName is not None:
+                subjectIndex = len(returnResults)
+                subjectNameJSON = {'Organization Name': subjectName,
+                                   'Entity Type': 'Organization'}
+                if subjectSerial is not None:
+                    subjectNameJSON['Registration Number'] = subjectSerial
+                tentativeSubjectResults.append([subjectNameJSON,
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we only have the serial of the subject, use that as the subject entity representation.
+            elif subjectSerial is not None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Phrase': subjectSerial,
+                                                 'Entity Type': 'Phrase'},
+                                                {uid: {'Resolution': 'Certificate Subject Serial Number',
+                                                       'Notes': ''}}])
+
+            # If we have no company name and no serial, the certificate belongs to an unknown org.
+            if subjectIndex is None:
+                subjectIndex = len(returnResults)
+                tentativeSubjectResults.append([{'Organization Name': 'Unknown Organization',
+                                                 'Entity Type': 'Organization'},
+                                                {uid: {'Resolution': 'Certificate Subject Organization',
+                                                       'Notes': ''}}])
+
+            # If we have an address, fill it in. If not, create separate entities for each element.
+            if streetAddr is not None:
+                streetAddressJSON = {'Street Address': streetAddr,
+                                     'Entity Type': 'Address'}
+                if subjectCountry is not None:
+                    streetAddressJSON['Country'] = subjectCountry
+                if postalCode is not None:
+                    streetAddressJSON['Postal Code'] = postalCode
+                if locality is not None:
+                    streetAddressJSON['Locality'] = locality
+                tentativeSubjectResults.append([streetAddressJSON,
+                                                {subjectIndex: {'Resolution': 'Certificate Subject Address',
+                                                                'Notes': ''}}])
+            else:
+                if subjectCountry is not None:
+                    tentativeSubjectResults.append([{'Country Name': subjectCountry,
+                                                     'Entity Type': 'Country'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Country',
+                                                                    'Notes': ''}}])
+                if postalCode is not None:
+                    tentativeSubjectResults.append([{'Phrase': postalCode,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Postal Code',
+                                                                    'Notes': ''}}])
+                if locality is not None:
+                    tentativeSubjectResults.append([{'Phrase': locality,
+                                                     'Entity Type': 'Phrase'},
+                                                    {subjectIndex: {'Resolution': 'Certificate Subject Locality',
+                                                                    'Notes': ''}}])
+            if len(tentativeSubjectResults) > 1:
+                # If we have more than just 'Unknown Organization' as a result for the subject, consider the output.
+                #   Otherwise, no point in including it.
+                returnResults += tentativeSubjectResults
+
+            # Validity dates for the certificate
+            returnResults.append([{'Date': websiteCertificate['notBefore'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Start Date',
+                                         'Notes': ''}}])
+            returnResults.append([{'Date': websiteCertificate['notAfter'],
+                                   'Entity Type': 'Date'},
+                                  {uid: {'Resolution': 'Certificate Expiry Date',
+                                         'Notes': ''}}])
+
+            # Domain names included in the certificate.
+            with contextlib.suppress(KeyError):
+                for altNameAttribute in websiteCertificate['subjectAltName']:
+                    returnResults.append([{'Domain Name': altNameAttribute[1],
+                                           'Entity Type': 'Domain'},
+                                          {uid: {'Resolution': 'Certificate Subject Alternate Name',
+                                                 'Notes': ''}}])
+            # OCSP URLs. Often just one.
+            with contextlib.suppress(KeyError):
+                for ocsp in websiteCertificate['OCSP']:
+                    returnResults.append([{'URL': ocsp,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate OCSP URL',
+                                                 'Notes': ''}}])
+            # CA Issuer URL
+            with contextlib.suppress(KeyError):
+                for caIssuer in websiteCertificate['caIssuers']:
+                    returnResults.append([{'URL': caIssuer,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Issuer URL',
+                                                 'Notes': ''}}])
+            # CRL URLs
+            with contextlib.suppress(KeyError):
+                for crlDistributionPoint in websiteCertificate['crlDistributionPoints']:
+                    returnResults.append([{'URL': crlDistributionPoint,
+                                           'Entity Type': 'Website'},
+                                          {uid: {'Resolution': 'Certificate Authority Revocation List URL',
+                                                 'Notes': ''}}])
+            # Issuer information
+            orgName = None
+            orgCommonName = None
+            orgCountry = None
+            orgPostal = None
+            orgLocality = None
+            orgStateOrProvince = None
+            orgUnitName = None
+            for issuerAttributeOuter in websiteCertificate['issuer']:
+                for issuerAttributeInner in issuerAttributeOuter:
+                    issuerAttributeInnerKey = issuerAttributeInner[0]
+                    issuerAttributeInnerValue = issuerAttributeInner[1]
+                    if issuerAttributeInnerKey == 'commonName':
+                        orgCommonName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'countryName':
+                        orgCountry = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'localityName':
+                        orgLocality = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'orgUnitName':
+                        orgUnitName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'organizationName':
+                        orgName = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'postalCode':
+                        orgPostal = issuerAttributeInnerValue
+                    elif issuerAttributeInnerKey == 'stateOrProvinceName':
+                        orgStateOrProvince = issuerAttributeInnerValue
+            issuerIndex = None
+            if orgName is not None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': orgName,
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Organization',
+                                             'Notes': ''}}])
+            if orgCommonName is not None:
+                if issuerIndex is not None:
+                    parentEntity = issuerIndex
+                else:
+                    parentEntity = uid
+                    issuerIndex = len(returnResults)
+                returnResults.append([{'Phrase': orgCommonName,
+                                       'Entity Type': 'Phrase'},
+                                      {parentEntity: {'Resolution': 'Certificate Issuer Common Name',
+                                                      'Notes': ''}}])
+            if issuerIndex is None:
+                issuerIndex = len(returnResults)
+                returnResults.append([{'Organization Name': 'Unknown Issuer',
+                                       'Entity Type': 'Organization'},
+                                      {uid: {'Resolution': 'Certificate Issuer Name',
+                                             'Notes': ''}}])
+            if orgCountry is not None:
+                returnResults.append([{'Country Name': orgCountry,
+                                       'Entity Type': 'Country'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Country',
+                                                     'Notes': ''}}])
+            if orgPostal is not None:
+                returnResults.append([{'Phrase': orgPostal,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Postal Code',
+                                                     'Notes': ''}}])
+            if orgLocality is not None:
+                returnResults.append([{'Phrase': orgLocality,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Locality',
+                                                     'Notes': ''}}])
+            if orgStateOrProvince is not None:
+                returnResults.append([{'Phrase': orgStateOrProvince,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer State / Province',
+                                                     'Notes': ''}}])
+            if orgUnitName is not None:
+                returnResults.append([{'Phrase': orgUnitName,
+                                       'Entity Type': 'Phrase'},
+                                      {issuerIndex: {'Resolution': 'Certificate Issuer Organizational Unit Name',
+                                                     'Notes': ''}}])
+
+            # Certificate info - serial number and certificate version.
+            returnResults.append([{'Phrase': websiteCertificate['serialNumber'],
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Serial Number',
+                                         'Notes': ''}}])
+
+            returnResults.append([{'Phrase': 'Certificate Version: ' + str(websiteCertificate['version']),
+                                   'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': 'Certificate Version',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ContainsPhrase.py

@@ -5,6 +5,8 @@ class ContainsPhrase:
     # A string that is treated as the name of this resolution.
     name = "Contains Phrase"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
     description = "Checks if a Phrase exists in the Notes of a Phrase entity or the body of a Website."
 
@@ -12,12 +14,16 @@ class ContainsPhrase:
 
     resultTypes = {'Phrase'}
 
-    parameters = {'Phrase to Search for': {'description': 'Please enter the Phrase to be searched for.',
+    parameters = {'Phrase to Search for': {'description': 'Please enter the Phrase to be searched for. Note that the '
+                                                          'search is performed in the "Notes" field of a Phrase or the '
+                                                          'body of the selected Website entities, not their primary '
+                                                          'fields.',
                                            'type': 'String',
                                            'value': ''},
                   'Case Sensitive': {'description': 'Do you want the phrase to be case sensitive?',
                                      'type': 'SingleChoice',
-                                     'value': {'Yes', 'No'}
+                                     'value': {'Yes', 'No'},
+                                     'default': 'Yes'
                                      }
                   }
 
@@ -26,7 +32,7 @@ class ContainsPhrase:
         import re
 
         headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
         }
 
         returnResults = []
@@ -37,9 +43,11 @@ class ContainsPhrase:
             searchPhrase = parameters['Phrase to Search for']
             uid = entity['uid']
             if entity['Entity Type'] == 'Phrase':
-                text = str(entity['Notes'])
+                text = str(entity.get('Notes'))
+                primaryField = entity['Phrase']
             else:  # Website entity
                 request = requests.get(entity["URL"], headers=headers)
+                primaryField = entity['URL']
                 text = request.text
 
             if parameters['Case Sensitive'] == 'No':
@@ -53,12 +61,13 @@ class ContainsPhrase:
                     offsets.append(match.start())
                     counter += 1
 
-            returnResults.append([{'Phrase': searchPhrase,
-                                   'Entity Type': 'Phrase',
-                                   'Notes': f'{searchPhrase} was found {counter} times\n'
-                                            f'offsets: Matches at character indices: '
-                                            f'{(", ".join(map(str, offsets)))}'},
-                                  {uid: {'Resolution': 'Contains Phrase',
-                                         'Notes': ''}}])
+            if counter > 0:
+                returnResults.append([{'Phrase': f'{primaryField} Contains Phrase: "{searchPhrase}" {counter} times',
+                                       'Entity Type': 'Phrase',
+                                       'Notes': f'"{searchPhrase}" was found {counter} time(s)\n'
+                                                f'Offsets: Matches at character indices: '
+                                                f'{(", ".join(map(str, offsets)))}'},
+                                      {uid: {'Resolution': f'Contains Phrase {searchPhrase}',
+                                             'Notes': ''}}])
 
         return returnResults

Core/Resolutions/Core/CryptoAddressExtractor.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+
+"""
+Credit to @cyb_detective:
+https://medium.com/@cyb_detective/20-regular-expressions-examples-to-search-for-data-related-to-cryptocurrencies-43e31dd4a5dc
+"""
+
+
+class CryptoAddressExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Cryptocurrency Addresses"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns patterns matching common cryptocurrency address formats on a website."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Crypto Wallet'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        from pathlib import Path
+        import re
+
+        playwrightPath = Path(parameters['Playwright Chromium'])
+        returnResults = []
+
+        ethRegex = re.compile(r"\b0[xX][a-fA-F0-9]{40}\b")
+        btcRegex = re.compile(
+            r"\b(?:bc(?:0(?:[ac-hj-np-z02-9]{39}|[ac-hj-np-z02-9]{59})|1[ac-hj-np-z02-9]{8,87})|[13][a-km-zA-HJ-NP-Z1-9]{25,34})\b")
+        bchRegex = re.compile(r"\b(?:(?:bitcoincash|bchreg|bchtest):)?[qp][a-z0-9]{41}\b")
+        moneroRegex = re.compile(r"\b[48][0-9AB][1-9A-HJ-NP-Za-km-z]{93}\b")
+        dogeRegex = re.compile(r"\bD[5-9A-HJ-NP-U][1-9A-HJ-NP-Za-km-z]{32}\b")
+        dashRegex = re.compile(r"\bX[1-9A-HJ-NP-Za-km-z]{33}\b")
+        rippleRegex = re.compile(r"\br[1-9A-HJ-NP-Za-km-z]{24,34}\b")
+        neoRegex = re.compile(r"\bN[0-9a-zA-Z]{33}\b")
+        litecoinRegex = re.compile(r"\b[LM3][a-km-zA-HJ-NP-Z1-9]{26,33}\b")
+        cosmosRegex = re.compile(r"\bcosmos[a-zA-Z0-9_.-]{10,}\b")
+        cardanoRegex = re.compile(r"\baddr1[a-z0-9]{10,}\b")
+        iotaRegex = re.compile(r"\biota[a-z0-9]{10,}\b")
+        liskRegex = re.compile(r"\b[0-9]{19}L\b")
+        nemRegex = re.compile(
+            r"\bN[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}-[A-Za-z0-9]{4,7}\b")
+        ontologyRegex = re.compile(r"\bA[0-9a-zA-Z]{33}\b")
+        polkadotRegex = re.compile(r"\b1[0-9a-zA-Z]{47}\b")
+        stellarRegex = re.compile(r"\bG[0-9A-Z]{55}\b")  # Stellar addresses are always 56 characters long.
+
+        # The software can deduplicate, but handling it here is better.
+        allWallets = set()
+
+        def extractCryptoAddresses(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            # Remove <span> and <noscript> tags.
+            while True:
+                try:
+                    soupContents.noscript.extract()
+                except AttributeError:
+                    break
+            while True:
+                try:
+                    soupContents.span.extract()
+                except AttributeError:
+                    break
+            siteContent = soupContents.get_text()
+
+            ethMatch = ethRegex.findall(siteContent)
+            for potentialMatch in ethMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Etherium',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Etherium Wallet Address',
+                                                        'Notes': ''}}])
+            btcMatch = btcRegex.findall(siteContent)
+            for potentialMatch in btcMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Bitcoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Bitcoin or Bitcoin Cash Wallet Address',
+                                                        'Notes': ''}}])
+            bchMatch = bchRegex.findall(siteContent)
+            for potentialMatch in bchMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Bitcoin Cash',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Bitcoin Cash Wallet Address',
+                                                        'Notes': ''}}])
+            xmrMatch = moneroRegex.findall(siteContent)
+            for potentialMatch in xmrMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Monero',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Monero Wallet Address',
+                                                        'Notes': ''}}])
+            dogeMatch = dogeRegex.findall(siteContent)
+            for potentialMatch in dogeMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Dogecoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Dogecoin Wallet Address',
+                                                        'Notes': ''}}])
+            dashMatch = dashRegex.findall(siteContent)
+            for potentialMatch in dashMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Dash',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Dash Wallet Address',
+                                                        'Notes': ''}}])
+            rippleMatch = rippleRegex.findall(siteContent)
+            for potentialMatch in rippleMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Ripple',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Ripple Wallet Address',
+                                                        'Notes': ''}}])
+            neoMatch = neoRegex.findall(siteContent)
+            for potentialMatch in neoMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Neo',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Neo Wallet Address',
+                                                        'Notes': ''}}])
+            litecoinMatch = litecoinRegex.findall(siteContent)
+            for potentialMatch in litecoinMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Litecoin',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Litecoin Wallet Address',
+                                                        'Notes': ''}}])
+            cosmosMatch = cosmosRegex.findall(siteContent)
+            for potentialMatch in cosmosMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Cosmos',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Cosmos Wallet Address',
+                                                        'Notes': ''}}])
+            cardanoMatch = cardanoRegex.findall(siteContent)
+            for potentialMatch in cardanoMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Cardano',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Cardano Wallet Address',
+                                                        'Notes': ''}}])
+            iotaMatch = iotaRegex.findall(siteContent)
+            for potentialMatch in iotaMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Iota',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Iota Wallet Address',
+                                                        'Notes': ''}}])
+            liskMatch = liskRegex.findall(siteContent)
+            for potentialMatch in liskMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Lisk',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Lisk Wallet Address',
+                                                        'Notes': ''}}])
+            nemMatch = nemRegex.findall(siteContent)
+            for potentialMatch in nemMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Nem',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Nem Wallet Address',
+                                                        'Notes': ''}}])
+            ontologyMatch = ontologyRegex.findall(siteContent)
+            for potentialMatch in ontologyMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Ontology',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Ontology Wallet Address',
+                                                        'Notes': ''}}])
+            polkadotMatch = polkadotRegex.findall(siteContent)
+            for potentialMatch in polkadotMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Polkadot',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Polkadot Wallet Address',
+                                                        'Notes': ''}}])
+            stellarMatch = stellarRegex.findall(siteContent)
+            for potentialMatch in stellarMatch:
+                if potentialMatch not in allWallets:
+                    allWallets.add(potentialMatch)
+                    returnResults.append([{'Wallet Address': potentialMatch,
+                                           'Currency Name': 'Stellar',
+                                           'Entity Type': 'Crypto Wallet'},
+                                          {currentUID: {'Resolution': 'Potential Stellar Wallet Address',
+                                                        'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(executable_path=playwrightPath)
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractCryptoAddresses(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/DecodePhrase.py

@@ -2,8 +2,10 @@ class DecodePhrase:
     # A string that is treated as the name of this resolution.
     name = "Decode Phrase"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
-    description = "Returns decoded phrase"
+    description = "Decode encoded phrases."
 
     # A set of entities that this resolution can be ran on.
     originTypes = {'Phrase'}
@@ -45,11 +47,10 @@ class DecodePhrase:
                 if len(text) % 8 != 0:
                     return "Malformed format not in Octaves"
 
-                ascii_string = ''
-                for binaryIndex in range(0, len(text), 8):
-                    ascii_string += chr(int(text[binaryIndex:binaryIndex + 8], 2))
-                returnResult.append([{'Phrase': str(ascii_string),
-                                      'Entity Type': 'Phrase'},
+                ascii_string = ''.join(chr(int(text[binaryIndex: binaryIndex + 8], 2))
+                                       for binaryIndex in range(0, len(text), 8))
+
+                returnResult.append([{'Phrase': ascii_string, 'Entity Type': 'Phrase'},
                                      {uid: {'Resolution': 'Binary Decoded Phrase', 'Notes': ''}}])
 
         return returnResult

Core/Resolutions/Core/DecodeRedirectUrlParameter.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+
+class DecodeRedirectUrlParameter:
+    name = "Decode Redirect URL"
+    category = "Website Information"
+    description = "Search the URL's parameters to see where you'll be redirected."
+    originTypes = {'Website'}
+    resultTypes = {'Website'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        from urllib.parse import urlparse
+        from urllib.parse import parse_qs
+        from urllib.parse import unquote
+        from base64 import b64decode
+
+        def get_redirect_value(parameter_arg: str) -> str:
+            with contextlib.suppress(Exception):
+                clean_val = unquote(parameter_arg)
+                if urlparse(clean_val).scheme:
+                    return clean_val
+                clean_val = unquote(b64decode(parameter_arg).decode('UTF-8'))
+                if urlparse(clean_val).scheme:
+                    return clean_val
+            return ''
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['URL'].strip()
+            parsed_url = urlparse(primaryField, allow_fragments=False)
+
+            parsed_url_params = parse_qs(parsed_url.query)
+            for param, param_value in parsed_url_params.items():
+                param_potential_url_value = get_redirect_value(', '.join(param_value))
+                if param_potential_url_value:
+                    parsed_url_params_copy = dict(parsed_url_params)
+                    parsed_url_params_copy.pop(param)
+                    new_entity = {'URL': param_potential_url_value,
+                                  'Entity Type': 'Website'}
+                    for param_copy, param_value_copy in parsed_url_params_copy.items():
+                        new_entity[param_copy] = ', '.join(param_value_copy)
+                    returnResults.append([new_entity,
+                                          {entity['uid']: {'Resolution': 'Redirect To',
+                                                           'Notes': ''}}])
+                    break
+
+        return returnResults

Core/Resolutions/Core/DeleteColumn.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+
+class DeleteColumn:
+    # A string that is treated as the name of this resolution.
+    name = "Rename or Delete Column"
+
+    category = "Spreadsheet Operations"
+
+    # A string that describes this resolution.
+    description = "Deletes the column with the specified index from a Spreadsheet document."
+
+    originTypes = {'Spreadsheet'}
+
+    resultTypes = {'Spreadsheet'}
+
+    parameters = {'Working Sheet': {'description': 'The name or index of the Sheet to read in the Spreadsheet '
+                                                   'file. By default, the first Sheet is used.',
+                                    'type': 'String',
+                                    'value': '0',
+                                    'default': '0'},
+                  'Column Name to Rename': {'description': 'Please enter the name of the column that you wish to  '
+                                                           'rename or delete.',
+                                            'type': 'String',
+                                            'value': ''},
+                  'New Column Name': {'description': 'Please enter the new name for the column.\nEnter the same name '
+                                                     'to delete the column instead.',
+                                      'type': 'String',
+                                      'value': ''}
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        from pathlib import Path
+        import pandas as pd
+        import contextlib
+
+        workingSheet = parameters['Working Sheet']
+        with contextlib.suppress(ValueError):
+            workingSheet = int(workingSheet)
+
+        renameColumn = parameters['Column Name to Rename']
+        targetColumn = parameters['New Column Name']
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            filePath = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not filePath.exists() or not filePath.is_file():
+                continue
+
+            try:
+                csvDF = pd.read_excel(filePath, sheet_name=workingSheet)
+            except ValueError:
+                continue
+
+            if renameColumn == targetColumn:
+                csvDF.drop(renameColumn, inplace=True)
+            else:
+                csvDF.rename(columns={renameColumn: targetColumn}, inplace=True)
+
+            count = 0
+            while True:
+                newFileName = f"{filePath.name.split(filePath.suffix, 1)[0]}-c{count}{filePath.suffix}"
+                newFilePath = filePath.parent / newFileName
+                if not newFilePath.exists():
+                    break
+            csvDF.to_excel(newFilePath, index=False)
+
+            returnResults.append([{'Spreadsheet Name': newFileName,
+                                   'File Path': newFileName,
+                                   'Entity Type': 'Spreadsheet'},
+                                  {uid: {'Resolution': 'Rename/Delete Column',
+                                         'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/DomainFromPhrase.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+
+class DomainFromPhrase:
+    name = "Domain From Phrase"
+    category = "String Operations"
+    description = "Extract domain-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+        import contextlib
+        import tldextract
+
+        domainRegex = re.compile(
+            r'^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|'
+            r'([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.'
+            r'([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$'
+        )
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase'].lower()
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if domainRegex.match(entityChunk):
+                    with contextlib.suppress(Exception):
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'Domain Name': entityChunk,
+                                                   'Entity Type': 'Domain'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Domain',
+                                                                   'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/EmailExtractor.py

@@ -5,6 +5,8 @@ class EmailExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Emails"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns the email addresses present on a website or index page of a domain."
 
@@ -12,18 +14,7 @@ class EmailExtractor:
 
     resultTypes = {'Email Address'}
 
-    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
-                                               'explored to discover more entities. Each entity discovered after '
-                                               'exploring sites linked in the original website or domain is said to '
-                                               'have a "depth" value of 1. Entities found from exploring the links on '
-                                               'this page would have a "depth" of 2, and so on. A larger value could '
-                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
-                                               'The default value is "0", which means only the provided website, or '
-                                               'the index page of the domain provided, is explored.',
-                                'type': 'String',
-                                'value': '0',
-                                'default': '0'},
-                  'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
+    parameters = {'Use Regex': {'description': 'Extraction of emails is done by finding "mailto" links in the source '
                                                'code of the website. However, not all emails on the site may exist in '
                                                'that format. Using Regex can result in more emails being extracted, '
                                                'however it is possible that some false positives may be extracted '
@@ -44,34 +35,29 @@ class EmailExtractor:
                   }
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
-        import tldextract
+        from pathlib import Path
         import re
+        import contextlib
         from email_validator import validate_email, caching_resolver, EmailNotValidError
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         returnResults = []
 
-        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
-        try:
-            maxDepth = max(int(parameters['Max Depth']), 0)
-        except ValueError:
-            return "Invalid value provided for Max Webpages to follow."
-
         # Source: https://emailregex.com/
         # Alt: (?:[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+(\.([a-zA-Z0-9-])+)+)
-        emailRegex = re.compile(r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""")
-        useRegex = True if parameters['Use Regex'] == 'Yes' else False
+        emailRegex = re.compile(
+            r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""")
+        useRegex = parameters['Use Regex'] == 'Yes'
 
         resolver = caching_resolver(timeout=10)
-        verifyDomain = True if parameters['Verify Email Domain Validity'] == 'Yes' else False
-
-        exploredDepth = set()
+        verifyDomain = parameters['Verify Email Domain Validity'] == 'Yes'
 
         # The software can deduplicate, but handling it here is better.
         allEmails = set()
 
-        def extractEmails(currentUID: str, site: str, depth: int):
+        def extractEmails(currentUID: str, site: str):
             page = context.new_page()
             pageResolved = False
             for _ in range(3):
@@ -81,71 +67,74 @@ class EmailExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
             soupContents = BeautifulSoup(page.content(), 'lxml')
             if useRegex:
-                potentialEmails = emailRegex.findall(soupContents.get_text())
-                for potentialEmail in potentialEmails:
+                # Remove <span> and <noscript> tags, and attempt some basic de-obfuscation.
+                while True:
                     try:
+                        soupContents.noscript.extract()
+                    except AttributeError:
+                        break
+                while True:
+                    try:
+                        soupContents.span.extract()
+                    except AttributeError:
+                        break
+                siteContent = soupContents.get_text()
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*at\s*(\]|\>|\))+\s*', '@', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*dot\s*(\]|\>|\))+\s*', '.', siteContent)
+                siteContent = re.sub(r'\s*(\[|\<|\()+\s*\.\s*(\]|\>|\))+\s*', '.', siteContent)
+                siteContent = re.sub(r'\s*@\s*', '@', siteContent)
+                siteContent = re.sub(r'\s*\.', '.', siteContent)
+                siteContent = re.sub(r'\.\s*([^A-Z])', r'.\1', siteContent)
+
+                potentialEmails = emailRegex.findall(siteContent)
+                for potentialEmail in potentialEmails:
+                    with contextlib.suppress(EmailNotValidError):
                         valid = validate_email(potentialEmail, dns_resolver=resolver, check_deliverability=verifyDomain)
-                        if valid.email not in allEmails:
-                            allEmails.add(valid.email)
-                            returnResults.append([{'Email Address': valid.email,
+                        if valid.normalized not in allEmails:
+                            allEmails.add(valid.normalized)
+                            returnResults.append([{'Email Address': valid.normalized,
                                                    'Entity Type': 'Email Address'},
                                                   {currentUID: {'Resolution': 'Email Address Found',
                                                                 'Notes': ''}}])
-                    except EmailNotValidError:
-                        pass
             linksInAHref = soupContents.find_all('a')
             for tag in linksInAHref:
                 newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('mailto:'):
-                        try:
-                            valid = validate_email(newLink[7:], dns_resolver=resolver,
-                                                   check_deliverability=verifyDomain)
-                            if valid.email not in allEmails:
-                                allEmails.add(valid.email)
-                                returnResults.append([{'Email Address': valid.email,
-                                                       'Entity Type': 'Email Address'},
-                                                      {currentUID: {'Resolution': 'Email Address Found',
-                                                                    'Notes': ''}}])
-                        except EmailNotValidError:
-                            pass
-                    elif newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractEmails(currentUID, newLink, newDepth)
-
-            linksInLinkHref = soupContents.find_all('link')
-            for tag in linksInLinkHref:
-                newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractEmails(currentUID, newLink, newDepth)
+                if newLink is not None and newLink.startswith('mailto:'):
+                    with contextlib.suppress(EmailNotValidError):
+                        valid = validate_email(newLink[7:], dns_resolver=resolver,
+                                               check_deliverability=verifyDomain)
+                        if valid.normalized not in allEmails:
+                            allEmails.add(valid.normalized)
+                            returnResults.append([{'Email Address': valid.normalized,
+                                                   'Entity Type': 'Email Address'},
+                                                  {currentUID: {'Resolution': 'Email Address Found',
+                                                                'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                viewport={'width': 1920, 'height': 1080}
             )
             for entity in entityJsonList:
                 uid = entity['uid']
-                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
-                domain = tldextract.extract(url).fqdn
-                extractEmails(uid, url, maxDepth)
+                    url = f'http://{url}'
+                extractEmails(uid, url)
             browser.close()
 
         return returnResults

Core/Resolutions/Core/EmailToDomain.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+class EmailToDomain:
+    name = "Email To Domain"
+    category = "Network Infrastructure"
+    description = "Get the Domain that an email address belongs to."
+    originTypes = {'Email Address'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            # There is no provider that I am aware of that allows '@' signs in the user part of the email.
+            with contextlib.suppress(Exception):
+                returnResults.append([{'Domain Name': primaryField.split('@')[1].strip(),
+                                       'Entity Type': 'Domain'},
+                                      {entity['uid']: {'Resolution': 'Email To Domain',
+                                                       'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/EmailToPhrase.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+
+class EmailToPhrase:
+    name = "Email Username To Phrase"
+    category = "String Operations"
+    description = "Get the username associated with the given email address."
+    originTypes = {'Email Address'}
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Email Address']
+            returnResults.append([{'Phrase': primaryField.split('@')[0],
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'Email Username To Phrase',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/ExtractDOCXMeta.py

@@ -4,9 +4,10 @@
 class ExtractDOCXMeta:
     # A string that is treated as the name of this resolution.
     name = "Get DOCX Metadata"
+    category = "File Operations"
 
     # A string that describes this resolution.
-    description = "Returns a set of nodes that contain all the metadata info of the file."
+    description = "Returns a set of nodes that contain all the metadata info of a docx file."
 
     originTypes = {'Document', 'Archive'}
 
@@ -25,7 +26,7 @@ class ExtractDOCXMeta:
             uid = entity['uid']
             filePath = Path(parameters['Project Files Directory']) / entity['File Path']
 
-            if not (filePath.exists() and filePath.is_file()):
+            if not filePath.exists() or not filePath.is_file():
                 continue
 
             if magic.from_file(str(filePath), mime=True) != \
@@ -48,9 +49,8 @@ class ExtractDOCXMeta:
                                    'Entity Type': 'Date'},
                                   {uid: {'Resolution': 'created', 'Notes': ''}}])
 
-            for metadataKey in [dataKey for dataKey in data if dataKey not in defaultDateProperties]:
-                returnResults.append([{'Phrase': metadataKey + ': ' + str(data.get(metadataKey)),
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+            returnResults.extend([{'Phrase': f'{metadataKey}: {str(data.get(metadataKey))}', 'Entity Type': 'Phrase'},
+                                  {uid: {'Resolution': metadataKey, 'Notes': ''}}] for metadataKey in
+                                 [dataKey for dataKey in data if dataKey not in defaultDateProperties])
 
         return returnResults

Core/Resolutions/Core/ExtractDateCreated.py

@@ -3,7 +3,8 @@
 
 class ExtractDateCreated:
     name = "Extract Date Created"
-    description = "Extract the Date Created field of selected entities"
+    category = "Date Operations"
+    description = "Extract the date in the Date Created field of the selected entities."
     originTypes = {'*'}
     resultTypes = {'Date'}
     parameters = {}

Core/Resolutions/Core/ExtractPDFMeta.py

@@ -4,9 +4,10 @@
 class ExtractPDFMeta:
     # A string that is treated as the name of this resolution.
     name = "Get PDF Metadata"
+    category = "File Operations"
 
     # A string that describes this resolution.
-    description = "Returns Nodes of contact info for websites"
+    description = "Returns a set of nodes that contain notable metadata info of pdf files."
 
     originTypes = {'Document'}
 
@@ -15,7 +16,8 @@ class ExtractPDFMeta:
     parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-        from PyPDF2 import PdfFileReader
+        from pypdf import PdfReader
+        from datetime import datetime, timedelta
         import magic
         from pathlib import Path
 
@@ -25,44 +27,55 @@ class ExtractPDFMeta:
             uid = entity['uid']
             filePath = Path(parameters['Project Files Directory']) / entity['File Path']
 
-            if not (filePath.exists() and filePath.is_file()):
+            if not (filePath.is_file()):
                 continue
 
-            if magic.from_file(str(filePath), mime=True) != \
-                    'application/pdf':
+            if magic.from_file(str(filePath), mime=True) != 'application/pdf':
                 continue
 
             with open(filePath, 'rb') as f:
-                pdf = PdfFileReader(f)
-                info = pdf.getDocumentInfo()
-                number_of_pages = pdf.getNumPages()
+                pdf = PdfReader(f)
+                info = pdf.metadata
+                number_of_pages = len(pdf.pages)
 
             for metadataKey in info:
+                attrValue = metadataKey[1:] if metadataKey.startswith('/') else metadataKey
                 if 'Date' in metadataKey:
                     try:
-                        strDate = info[metadataKey]
-                        strDate = strDate.split(':')[1].split('-')[0]
-                        strDate1 = strDate[:-6]
-                        strDate2 = strDate[-6:]
-                        strDate2 = ':'.join(strDate2[i:i+2] for i in range(0, 6, 2))
-                        strDate1 = strDate1[:-4] + '-' + '-'.join(strDate1[::-1][i:i+2] for i in range(0, 4, 2))[::-1]
-                        strDate = strDate1 + 'T' + strDate2
-                        returnResults.append([{'Date': strDate,
+                        strDate = info[metadataKey].split(':', 1)[1]
+                        if strDate.endswith('Z'):
+                            dateString = datetime.strptime(strDate, "%Y%m%d%H%M%SZ").isoformat()
+                        elif '+' in strDate:
+                            datePart1, datePart2 = strDate.split('+', 1)
+                            date1 = datetime.strptime(datePart1, "%Y%m%d%H%M%S")
+                            date2 = timedelta(hours=int(datePart2.split("'")[0]), minutes=int(datePart2.split("'")[1]))
+                            dateString = (date1 + date2).isoformat()
+                        elif '-' in strDate:
+                            datePart1, datePart2 = strDate.split('-', 1)
+                            date1 = datetime.strptime(datePart1, "%Y%m%d%H%M%S")
+                            date2 = timedelta(hours=int(datePart2.split("'")[0]), minutes=int(datePart2.split("'")[1]))
+                            dateString = (date1 - date2).isoformat()
+                        else:
+                            raise ValueError('Cannot parse Date format.')
+
+                        returnResults.append([{'Date': dateString,
                                                'Entity Type': 'Date'},
-                                              {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+                                              {uid: {'Resolution': attrValue, 'Notes': ''}}])
                     except Exception:
                         # Reset strDate to default value
                         strDate = info[metadataKey]
                         returnResults.append([{'Date': strDate,
                                                'Entity Type': 'Date'},
-                                              {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+                                              {uid: {'Resolution': attrValue, 'Notes': ''}}])
                 else:
-                    returnResults.append([{'Phrase': metadataKey + ': ' + str(info[metadataKey]),
+                    # Clean some misshapen strings
+                    value = str(info[metadataKey])
+                    value = value.removeprefix('/')
+                    returnResults.append([{'Phrase': f'{attrValue}: {value}',
                                            'Entity Type': 'Phrase'},
-                                          {uid: {'Resolution': metadataKey, 'Notes': ''}}])
+                                          {uid: {'Resolution': attrValue, 'Notes': ''}}])
 
-            returnResults.append([{'Phrase': 'Number of Pages: ' + str(number_of_pages),
-                                   'Entity Type': 'Phrase'},
+            returnResults.append([{'Phrase': f'Number of Pages: {number_of_pages}', 'Entity Type': 'Phrase'},
                                   {uid: {'Resolution': 'Number of Pages', 'Notes': ''}}])
 
         return returnResults

Core/Resolutions/Core/FileExtractor.py

@@ -19,12 +19,14 @@ class FileExtractor:
     # A string that is treated as the name of this resolution.
     name = "Find Hosted File URLs"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns Nodes of files in websites and domains."
 
     originTypes = {'Domain', 'Website'}
 
-    resultTypes = {'Website', 'Document', 'Image', 'Video', 'Archive'}
+    resultTypes = {'Website', 'Document', 'Spreadsheet', 'Image', 'Video', 'Archive'}
 
     parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
                                                'explored to discover more entities. Each entity discovered after '
@@ -39,27 +41,30 @@ class FileExtractor:
                                 'default': '0'}}
 
     def resolution(self, entityJsonList, parameters):
+        import contextlib
         import tldextract
         import requests
         from hashlib import md5
-        from binascii import hexlify
         from pathlib import Path
         from bs4 import BeautifulSoup
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+
+        playwrightPath = Path(parameters['Playwright Firefox'])
 
         try:
             maxDepth = max(int(parameters['Max Depth']), 0)
         except ValueError:
             return "Invalid value provided for Max Webpages to follow."
 
-        fileTypes = (".sxw", ".odt", ".ods", ".odg", ".odp", ".docx", ".xlsx", ".pptx", ".ppsx", ".doc", ".xls",
+        fileTypes = (".sxw", ".odt", ".odg", ".odp", ".docx", ".pptx", ".ppsx", ".doc", ".csv",
                      ".ppt", ".pps", ".pdf", ".wpd", ".raw", ".cr2", ".crw", ".indd", ".rdp", ".ica", ".ico", ".txt",
-                     ".text", ".bak", ".log", ".env", ".pub", ".docm", ".xlsm", ".old", ".csv", ".apk", ".sql", ".cfg",
+                     ".text", ".bak", ".log", ".env", ".pub", ".docm", ".old", ".apk", ".sql", ".cfg",
                      ".key", ".reg", ".yml", ".yaml", ".mail", ".eml", ".mbox", ".mbx", ".url", ".csr", ".config",
                      ".mdb", ".user", ".adr", ".ini", ".plist", ".conf", ".dat", ".pcf", ".bok", ".properties", ".json",
-                     ".backup", ".sh", ".py", ".md", ".inc")
-        videoTypes = (".mp3", ".mp4")
-        imageTypes = (".jpg", ".jpeg", ".png", ".svg", ".svgz")
+                     ".backup", ".sh", ".py", ".md", ".inc", '.ovpn', '.bat')
+        spreadsheetTypes = (".xlsx", ".xls", ".ods", ".xlsm")
+        videoTypes = (".mp3", ".mp4", ".mov", ".webm", ".amv")
+        imageTypes = (".jpg", ".jpeg", ".png", ".svg", ".svgz", ".bmp")
         archiveTypes = (".zip", ".rar", ".7z", ".gz")
 
         returnResults = []
@@ -74,6 +79,8 @@ class FileExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
 
             soupContents = BeautifulSoup(page.content(), 'lxml')
 
@@ -83,7 +90,7 @@ class FileExtractor:
                 if link is not None:
                     if not link.startswith('http'):
                         # We assume that we will be redirected to https if available.
-                        link = 'http://' + domain + link
+                        link = f'http://{domain}{link}'
                     link = link.split('#')[0]
                     if link not in urlsExplored:
                         urlsExplored.add(link)
@@ -99,6 +106,8 @@ class FileExtractor:
                             fileTypeIdentified = 'Image'
                         elif link.endswith(archiveTypes):
                             fileTypeIdentified = 'Archive'
+                        elif link.endswith(spreadsheetTypes):
+                            fileTypeIdentified = 'Spreadsheet'
 
                         if fileTypeIdentified:
                             childIndex = len(returnResults)
@@ -109,10 +118,10 @@ class FileExtractor:
                                                          'Notes': ''}}])
 
                             docProperName = link.split('/')[-1]
-                            docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
+                            docFileName = f'{md5(link.encode("UTF-8")).hexdigest()}_{docProperName}'
                             docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
-                            try:
+                            with contextlib.suppress(Exception):
                                 response = requests.get(link,
                                                         headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
                                                                                'x64; rv:94.0) Gecko/20100101 '
@@ -122,13 +131,10 @@ class FileExtractor:
                                     for chunk in response.iter_content(4096):
                                         fileToWrite.write(chunk)
 
-                                returnResults.append([{fileTypeIdentified + ' Name': docProperName,
+                                returnResults.append([{f'{fileTypeIdentified} Name': docProperName,
                                                        'File Path': docFileName,
                                                        'Entity Type': fileTypeIdentified},
-                                                      {childIndex: {'Resolution': 'Downloaded File',
-                                                                    'Notes': ''}}])
-                            except Exception:
-                                pass
+                                                      {childIndex: {'Resolution': 'Downloaded File', 'Notes': ''}}])
 
                         elif domain in link:
                             urlsToExplore.add(link)
@@ -139,7 +145,7 @@ class FileExtractor:
                 if link is not None:
                     if not link.startswith('http'):
                         # We assume that we will be redirected to https if available.
-                        link = 'http://' + domain + link
+                        link = f'http://{domain}{link}'
                     link = link.split('#')[0]
                     if link not in urlsExplored:
                         urlsExplored.add(link)
@@ -151,10 +157,10 @@ class FileExtractor:
                                               {uid: {'Resolution': 'File URL',
                                                      'Notes': ''}}])
                         docProperName = link.split('/')[-1]
-                        docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
+                        docFileName = f'{md5(link.encode()).hexdigest()}_{docProperName}'
                         docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
-                        try:
+                        with contextlib.suppress(Exception):
                             response = requests.get(link,
                                                     headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
                                                                            'x64; rv:94.0) Gecko/20100101 '
@@ -169,16 +175,13 @@ class FileExtractor:
                                                    'Entity Type': 'Image'},
                                                   {childIndex: {'Resolution': 'Downloaded File',
                                                                 'Notes': ''}}])
-                        except Exception:
-                            pass
-
             if currentDepth > 0:
                 newDepth = currentDepth - 1
                 for newURL in urlsToExplore:
                     iterateOnDepth(newURL, newDepth)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
@@ -190,7 +193,7 @@ class FileExtractor:
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
+                    url = f'http://{url}'
                 domain = tldextract.extract(url).fqdn
 
                 # Because these do not persist across entities, it is possible to explore a URL multiple times.

Core/Resolutions/Core/FileHasher.py

@@ -3,31 +3,32 @@
 
 class FileHasher:
     name = "Get File Hash"
-    description = "Get the Hash of a file"
-    originTypes = {"Image", "Document", "Video", "Archive", "Disk"}
+    category = "File Operations"
+    description = "Get the Hash of a file."
+    originTypes = {"Image", "Document", "Spreadsheet", "Video", "Archive", "Disk"}
     resultTypes = {'Hash'}
-    parameters = {'hashing_algorithms': {'description': 'The type of hash/es that will be returned',
-                                         'type': 'MultiChoice',
-                                         'value': {'SHA1', 'SHA256', 'MD5'}
-                                         }}
+    parameters = {'Hashing Algorithm': {'description': 'Choose the type of hash(es) that you want to be returned:',
+                                        'type': 'MultiChoice',
+                                        'value': {'SHA1', 'SHA256', 'MD5'}
+                                        }}
 
     def resolution(self, entityJsonList, parameters):
         import hashlib
         from pathlib import Path
 
         return_result = []
-        hashing_algorithms = parameters['hashing_algorithms']
+        hashing_algorithms = parameters['Hashing Algorithm']
         for entity in entityJsonList:
             uid = entity['uid']
-            file_path = Path(entity["File Path"])
-            if not (file_path.exists() and file_path.is_file()):
+            file_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not file_path.is_file():
                 continue
             block_size = 65536  # The size of each read from the file
             for hashing_algorithm in hashing_algorithms:
-                if hashing_algorithm == "SHA256":
-                    file_hash = hashlib.sha256()  # nosec
-                elif hashing_algorithm == "SHA1":
+                if hashing_algorithm == "SHA1":
                     file_hash = hashlib.sha1()  # nosec
+                elif hashing_algorithm == "SHA256":
+                    file_hash = hashlib.sha256()  # nosec
                 else:
                     file_hash = hashlib.md5()  # nosec
                 with open(file_path, 'rb') as f:
@@ -39,5 +40,5 @@ class FileHasher:
                 return_result.append([{'Hash Value': resulting_hash,
                                        'Hash Algorithm': hashing_algorithm,
                                        'Entity Type': 'Hash'},
-                                      {uid: {'Resolution': hashing_algorithm + ' Hash', 'Notes': ''}}])
+                                      {uid: {'Resolution': f'{hashing_algorithm} Hash', 'Notes': ''}}])
         return return_result

Core/Resolutions/Core/GetExternalURLs.py

@@ -18,9 +18,10 @@ Tweaks to this script can be made depending on each investigator's workflow.
 
 class GetExternalURLs:
     name = "Get External Urls"
+    category = "Website Information"
     description = "Returns all links to external sites on a website."
     originTypes = {'Website'}
-    resultTypes = {'Website'}
+    resultTypes = {'Website', 'Onion Website'}
 
     parameters = {'Element types to check': {'description': 'Select the types of elements to investigate for '
                                                             'external links. Note that "a" elements have the lowest '
@@ -35,37 +36,51 @@ class GetExternalURLs:
 
     def resolution(self, entityJsonList, parameters):
         import tldextract
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
         import re
-        import urllib
+        import contextlib
+        from urllib.parse import urlparse
+        from urllib.parse import parse_qs
+        from urllib.parse import unquote
+        from base64 import b64decode
+        from pathlib import Path
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
+        onionRegex = re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$""")
         returnResult = []
 
         extract_a = '<a> elements' in parameters['Element types to check']
         extract_img = '<img> elements' in parameters['Element types to check']
         extract_link = '<link> elements' in parameters['Element types to check']
 
-        # Sites like youtube replace external links with a redirect link originating
-        #   from the site itself. This sort of gets around that.
-        redirectRegex = re.compile(r'q=[^\s][^&^#]*', re.IGNORECASE)
+        def get_potential_redirect_value(potential_redirect_url: str) -> str:
+            parsed_url = urlparse(potential_redirect_url, allow_fragments=False)
+            parsed_url_params = parse_qs(parsed_url.query)
+            for param, param_value in parsed_url_params.items():
+                with contextlib.suppress(Exception):
+                    clean_val = unquote(', '.join(param_value))
+                    if urlparse(clean_val).scheme:
+                        return clean_val
+                    clean_val = unquote(b64decode(', '.join(param_value)).decode('UTF-8'))
+                    if urlparse(clean_val).scheme:
+                        return clean_val
+            return ''
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                viewport={'width': 1920, 'height': 1080}
             )
-            allPages = []
             page = context.new_page()
+            externalUrls = {}
 
             for site in entityJsonList:
                 uid = site['uid']
                 url = site['URL']
-                if url is None:
+                parsedURL = urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
                     continue
-                if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
                 domain = tldextract.extract(url).fqdn
 
                 # Try to load the page a few times, in case of timeouts.
@@ -73,50 +88,75 @@ class GetExternalURLs:
                 for _ in range(3):
                     try:
                         page.goto(url, wait_until="networkidle", timeout=10000)
-                        allPages.append((domain, page, uid))
+
+                        ### Youtube
+                        with contextlib.suppress(Exception):
+                            page.get_by_role("button",
+                                             name="Reject the use of cookies and other data for the purposes described").click()
+                        with contextlib.suppress(Exception):
+                            page.get_by_role("button", name="Show more").click()
+                        soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                        if extract_a:
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                parsedURL = urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]):
+                                    if domain in link:
+                                        redirectLink = get_potential_redirect_value(link)
+                                        if redirectLink:
+                                            try:
+                                                externalUrls[redirectLink].add(uid)
+                                            except KeyError:
+                                                externalUrls[redirectLink] = {uid}
+                                    else:
+                                        newLink = link.split('#')[0].split('?')[0]
+                                        try:
+                                            externalUrls[newLink].add(uid)
+                                        except KeyError:
+                                            externalUrls[newLink] = {uid}
+                        if extract_img:
+                            linksInImgSrc = soupContents.find_all('img')
+                            for tag in linksInImgSrc:
+                                link = tag.get('src', None)
+                                parsedURL = urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
+                                    newLink = link.split('#')[0].split('?')[0]
+                                    try:
+                                        externalUrls[newLink].add(uid)
+                                    except KeyError:
+                                        externalUrls[newLink] = {uid}
+
+                        if extract_link:
+                            linksInLinkHref = soupContents.find_all('link')
+                            for tag in linksInLinkHref:
+                                link = tag.get('href', None)
+                                parsedURL = urlparse(link)
+                                if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
+                                    newLink = link.split('#')[0].split('?')[0]
+                                    try:
+                                        externalUrls[newLink].add(uid)
+                                    except KeyError:
+                                        externalUrls[newLink] = {uid}
                         break
                     except TimeoutError:
                         pass
-            for urlVisited in allPages:
-                externalUrls = set()
-                soupContents = BeautifulSoup(urlVisited[1].content(), 'lxml')
+                    except Error:
+                        break
 
-                if extract_a:
-                    linksInAHref = soupContents.find_all('a')
-                    for tag in linksInAHref:
-                        link = tag.get('href', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-                                else:
-                                    redirLinks = redirectRegex.findall(link)
-                                    if 'redirect' in link and len(redirLinks) > 0:
-                                        newLink = str(urllib.parse.unquote(redirLinks[0]))[2:]
-                                        externalUrls.add(newLink)
-
-                if extract_img:
-                    linksInImgSrc = soupContents.find_all('img')
-                    for tag in linksInImgSrc:
-                        link = tag.get('src', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-
-                if extract_link:
-                    linksInLinkHref = soupContents.find_all('link')
-                    for tag in linksInLinkHref:
-                        link = tag.get('href', None)
-                        if link is not None:
-                            if link.startswith('http'):
-                                if urlVisited[0] not in link:
-                                    externalUrls.add(link.split('#')[0])
-
-                for externalUrl in externalUrls:
-                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
-                                         {urlVisited[2]: {'Resolution': 'External Link', 'Notes': ''}}])
             page.close()
+            context.close()
             browser.close()
 
+        for externalUrl in externalUrls:
+            onionCheck = onionRegex.findall(externalUrl)
+            for urlUid in externalUrls[externalUrl]:
+                if len(onionCheck) == 1:
+                    returnResult.append([{'Onion URL': externalUrl, 'Entity Type': 'Onion Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+                else:
+                    returnResult.append([{'URL': externalUrl, 'Entity Type': 'Website'},
+                                         {urlUid: {'Resolution': 'External Link', 'Notes': ''}}])
+
         return returnResult

Core/Resolutions/Core/GetInternalURLs.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+
+class GetInternalURLs:
+    name = "Get Domain Urls"
+    category = "Website Information"
+    description = "Returns all URLs in a Domain."
+    originTypes = {'Website', 'Domain'}
+    resultTypes = {'Website'}
+
+    parameters = {'Include Resources': {'description': 'Should the resolution also return links to images and internal '
+                                                       'links (i.e. links to resources used by the site)? Note that '
+                                                       'doing so will make the resolution take more time to complete.',
+                                        'type': 'SingleChoice',
+                                        'value': {'Include images and internal links', 'Only consider links to pages'},
+                                        'default': 'Only consider links to pages'}}
+
+    def resolution(self, entityJsonList, parameters):
+        import tldextract
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        from pathlib import Path
+        import urllib.parse
+
+        playwrightPath = Path(parameters['Playwright Chromium'])
+        returnResult = []
+        internalUrls = {}
+
+        considerResources = parameters['Include Resources'] != 'Only consider links to pages'
+
+        def handleLink(currentLink, currentUrl, currentDomain):
+            if currentLink is None:
+                return None
+            if currentLink.startswith('//'):
+                if currentUrl.endswith('/'):
+                    currentUrl = currentUrl[:-1]
+                currentLink = currentUrl + currentLink[1:]
+            elif currentLink.startswith('/'):
+                urlParts = urllib.parse.urlparse(currentUrl)
+                currentLink = f'{urlParts.scheme}://{urlParts.netloc}{currentLink}'
+            parsedCurrentURL = urllib.parse.urlparse(currentLink)
+            if (
+                all([parsedCurrentURL.scheme, parsedCurrentURL.netloc])
+                and currentDomain in currentLink
+            ):
+                newLink = currentLink.split('#')[0].split('?')[0]
+                if newLink.endswith('/'):
+                    newLink = newLink[:-1]
+                return newLink
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(executable_path=playwrightPath)
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                entityType = site['Entity Type']
+                if entityType == 'Domain':
+                    url = 'http://' + site['Domain Name']
+                else:
+                    url = site['URL']
+                parsedURL = urllib.parse.urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+                domain = tldextract.extract(url).fqdn
+
+                domainUrls = [url]
+                index = 0
+                while index < len(domainUrls):
+                    url = domainUrls[index]
+                    index += 1
+                    for _ in range(3):
+                        try:
+                            page.goto(url, wait_until="networkidle", timeout=10000)
+                            soupContents = BeautifulSoup(page.content(), 'lxml')
+
+                            if domain not in page.url:
+                                # Handle redirects to other sites
+                                try:
+                                    internalUrls[page.url].add(uid)
+                                except KeyError:
+                                    internalUrls[page.url] = {uid}
+                                break
+
+                            linksInAHref = soupContents.find_all('a')
+                            for tag in linksInAHref:
+                                link = tag.get('href', None)
+                                handledLink = handleLink(link, url, domain)
+                                if handledLink:
+                                    try:
+                                        internalUrls[handledLink].add(uid)
+                                    except KeyError:
+                                        internalUrls[handledLink] = {uid}
+                                    if handledLink not in domainUrls:
+                                        domainUrls.append(handledLink)
+
+                            if considerResources:
+                                linksInImgSrc = soupContents.find_all('img')
+                                for tag in linksInImgSrc:
+                                    link = tag.get('src', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+
+                                linksInLinkHref = soupContents.find_all('link')
+                                for tag in linksInLinkHref:
+                                    link = tag.get('href', None)
+                                    handledLink = handleLink(link, url, domain)
+                                    if handledLink:
+                                        try:
+                                            internalUrls[handledLink].add(uid)
+                                        except KeyError:
+                                            internalUrls[handledLink] = {uid}
+                                        if handledLink not in domainUrls:
+                                            domainUrls.append(handledLink)
+                            break
+                        except TimeoutError:
+                            pass
+                        except Error:
+                            break
+
+            page.close()
+            context.close()
+            browser.close()
+
+        for internalUrl in internalUrls:
+            for urlUid in internalUrls[internalUrl]:
+                returnResult.append([{'URL': internalUrl, 'Entity Type': 'Website'},
+                                     {urlUid: {'Resolution': 'Internal Link', 'Notes': ''}}])
+
+        return returnResult

Core/Resolutions/Core/GetWebsiteBody.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetWebsiteBody:
-    # A string that is treated as the name of this resolution.
-    name = "Get Website Body"
-
-    # A string that describes this resolution.
-    description = "Returns body of websites"
-
-    originTypes = {'Website'}
-
-    resultTypes = {'Phrase'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-
-        headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
-        }
-
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-
-            primaryField = entity[list(entity)[1]]
-
-            if primaryField.startswith('http://') or primaryField.startswith('https://'):
-                url = primaryField
-            else:
-                url = 'http://' + primaryField
-
-            r = requests.get(url, headers=headers)
-            doc = r.text
-
-            returnResults.append([{'Phrase': 'Website Body: ' + primaryField,
-                                   'Notes': doc,
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Website Body', 'Name': 'Website Body', 'Notes': ''}}])
-        return returnResults

Core/Resolutions/Core/GetWebsiteText.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+
+class GetWebsiteText:
+    # A string that is treated as the name of this resolution.
+    name = "Get Website Text"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns the text content of the selected websites."
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from bs4 import BeautifulSoup
+        from bs4.element import Comment
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from urllib.parse import urlparse
+        from pathlib import Path
+
+        returnResults = []
+        playwrightPath = Path(parameters['Playwright Chromium'])
+
+        def tag_visible(element):
+            if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
+                return False
+            return not isinstance(element, Comment)
+
+        def text_from_html(body):
+            soup = BeautifulSoup(body, 'lxml')
+            texts = soup.findAll(text=True)
+            visible_texts = filter(tag_visible, texts)
+            return u" ".join(t.strip() for t in visible_texts if t.strip() != '')
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(executable_path=playwrightPath)
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.64 Safari/537.36'
+            )
+            page = context.new_page()
+
+            for site in entityJsonList:
+                uid = site['uid']
+                url = site['URL']
+                parsedURL = urlparse(url)
+                if not all([parsedURL.scheme, parsedURL.netloc]):
+                    continue
+
+                # Try to load the page a few times, in case of timeouts.
+                # I don't think making parts of this async actually helps in this case.
+                for _ in range(3):
+                    try:
+                        page.goto(url, wait_until="networkidle", timeout=10000)
+                        textContent = text_from_html(page.content())
+                        returnResults.append([{'Phrase': f'Website Body of: {url}',
+                                               'Notes': textContent,
+                                               'Entity Type': 'Phrase'},
+                                              {uid: {'Resolution': 'Website Body', 'Notes': ''}}])
+                        break
+                    except TimeoutError:
+                        pass
+                    except Error:
+                        break
+
+        return returnResults

Core/Resolutions/Core/HostnameToDomain.py

@@ -2,8 +2,9 @@
 
 
 class HostnameToDomain:
-    name = "Hostname to Domain"
-    description = "Find the domain of any given hostname"
+    name = "Get Top Level Domain"
+    category = "Network Infrastructure"
+    description = "Get the top level domain of any given website or subdomain."
     originTypes = {'Domain', 'Website'}
     resultTypes = {'Domain'}
     parameters = {}
@@ -15,8 +16,7 @@ class HostnameToDomain:
         for entity in entityJsonList:
             uid = entity['uid']
             primary_field = entity[list(entity)[1]].strip()
-            tsd, td, tsu = extract(primary_field)
-            domain = td + '.' + tsu
+            domain = extract(primary_field).fqdn
             if domain == primary_field:
                 continue
             return_result.append([{'Domain Name': domain,

Core/Resolutions/Core/HostnameToIP.py

@@ -3,6 +3,7 @@
 
 class HostnameToIP:
     name = "Hostname To IP"
+    category = "Network Infrastructure"
     description = "Gets the IP associated with the given hostname."
     originTypes = {'Domain', 'Website'}
     resultTypes = {'IP Address', 'IPv6 Address'}

Core/Resolutions/Core/IPToASN.py

@@ -2,8 +2,10 @@ class IPToASN:
     # A string that is treated as the name of this resolution.
     name = "Get ASN From IP"
 
+    category = "Network Infrastructure"
+
     # A string that describes this resolution.
-    description = "IP to ASN"
+    description = "Get the Autonomous System Number that the selected IP Addresses belong to."
 
     # A set of entities that this resolution can be ran on.
     originTypes = {'IP Address'}
@@ -33,19 +35,19 @@ class IPToASN:
                 index_of_child = len(returnResult)
                 countryCode = results['asn_country_code']
                 country = pycountry.countries.get(alpha_2=countryCode).name
-                returnResult.append([{'AS Number': "AS" + results['asn'],
-                                      'ASN Cidr': results['asn_cidr'],
-                                      'Date Created': results['asn_date'],
-                                      'Entity Type': 'Autonomous System'},
-                                     {uid: {'Resolution': 'Autonomous System Results', 'Notes': ''}}])
-                returnResult.append(
-                    [{'Organization Name': results['asn_registry'], 'Entity Type': 'Organization'},
-                     {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}])
-                returnResult.append(
-                    [{'Country Name': country, 'Entity Type': 'Country'},
-                     {index_of_child: {'Resolution': 'Country of Registry for ASN', 'Notes': ''}}])
-                returnResult.append(
-                    [{'Phrase': results['asn_description'], 'Entity Type': 'Phrase'},
-                     {index_of_child: {'Resolution': 'ASN Description', 'Notes': ''}}])
+                returnResult.extend(([{'AS Number': "AS" + results['asn'],
+                                       'ASN Cidr': results['asn_cidr'],
+                                       'Date Created': results['asn_date'],
+                                       'Entity Type': 'Autonomous System'},
+                                      {uid: {'Resolution': 'Autonomous System of IP', 'Notes': ''}}],
+                                     [{'Organization Name': results['asn_registry'],
+                                       'Entity Type': 'Organization'},
+                                      {index_of_child: {'Resolution': 'ASN Registry', 'Notes': ''}}],
+                                     [{'Country Name': country,
+                                       'Entity Type': 'Country'},
+                                      {index_of_child: {'Resolution': 'Country of Registry for ASN', 'Notes': ''}}],
+                                     [{'Phrase': results['asn_description'],
+                                       'Entity Type': 'Phrase'},
+                                      {index_of_child: {'Resolution': 'ASN Description', 'Notes': ''}}]))
 
         return returnResult

Core/Resolutions/Core/IPToHostname.py

@@ -3,7 +3,8 @@
 
 class IPToHostname:
     name = "IP To Hostname"
-    description = "Gets the Fully Qualified Domain Name associated with the given IP / IPv6 address."
+    category = "Network Infrastructure"
+    description = "Gets the Fully Qualified Domain Name associated with the given IPv4 / IPv6 address."
     originTypes = {'IPv6 Address', 'IP Address'}
     resultTypes = {'Domain'}
 

Core/Resolutions/Core/IPWhois.py

@@ -3,7 +3,8 @@
 
 class IPWhois:
     name = "IPv4 WhoIs Information"
-    description = "Find information about the Whois of a given IP Address"
+    category = "Network Infrastructure"
+    description = "Extract information from the Whois record that corresponds to a given IP Address."
     originTypes = {'IP Address'}
     resultTypes = {'Country', 'Autonomous System', 'Email Address'}
     parameters = {}
@@ -24,14 +25,18 @@ class IPWhois:
                 return "The Entity Provided isn't a valid IP Address"
             IPobject = IPWhois(primary_field)
             response = IPobject.lookup_whois()
-            return_result.append([{'AS Number': str(response['asn']),
+            return_result.append([{'AS Number': 'AS' + str(response['asn']),
                                    'ASN Cidr': str(response['asn_cidr']),
                                    'Entity Type': 'Autonomous System'},
                                   {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
             for net in response['nets']:
                 if net['country'] is not None:
-                    country = pycountry.countries.get(alpha_2=net['country']).name
-                    return_result.append([{'Country Name': country,
+                    if country := pycountry.countries.get(alpha_2=net['country']):
+                        country_name = country.name
+                    else:
+                        # May not always be an actual Country.
+                        country_name = net['country']
+                    return_result.append([{'Country Name': country_name,
                                            'Entity Type': 'Country'},
                                           {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
                 if net['name'] is not None:
@@ -39,8 +44,6 @@ class IPWhois:
                                            'Entity Type': 'Company'},
                                           {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
                 if net['emails'] is not None:
-                    for email in net['emails']:
-                        return_result.append([{'Email Address': email,
-                                               'Entity Type': 'Email Address'},
-                                              {uid: {'Resolution': 'IPWhois', 'Notes': ''}}])
+                    return_result.extend([{'Email Address': email, 'Entity Type': 'Email Address'},
+                                          {uid: {'Resolution': 'IPWhois', 'Notes': ''}}] for email in net['emails'])
         return return_result

Core/Resolutions/Core/ImageToDevice.py

@@ -3,7 +3,8 @@
 
 class ImageToDevice:
     name = "Device From Image"
-    description = "Find information about on what device the image was taken"
+    category = "File Operations"
+    description = "Find information about on what device an image was taken with."
     originTypes = {"Image"}
     resultTypes = {'Phrase'}
     parameters = {}
@@ -16,23 +17,22 @@ class ImageToDevice:
         for entity in entityJsonList:
             uid = entity['uid']
             index_of_child = len(return_result)
-            image_path = Path(entity["File Path"])
-            if not (image_path.exists() and image_path.is_file()):
+            image_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not image_path.exists() or not image_path.is_file():
                 continue
             with open(image_path, 'rb') as image_file:
                 my_image = Image(image_file)
                 if my_image.has_exif is False:
                     continue
-                else:
-                    for tag in my_image.list_all():
-                        if tag == "make":
-                            return_result.append([{'Phrase': my_image.make,
-                                                   'Entity Type': 'Phrase'},
-                                                  {uid: {'Resolution': 'ExifMetadata Device Manufacturer',
-                                                         'Notes': ''}}])
-                        if tag == "model":
-                            return_result.append([{'Phrase': my_image.model,
-                                                   'Entity Type': 'Phrase'},
-                                                  {index_of_child: {'Resolution': 'ExifMetadata Device Model',
-                                                                    'Notes': ''}}])
+                for tag in my_image.list_all():
+                    if tag == "make":
+                        return_result.append([{'Phrase': my_image.make,
+                                               'Entity Type': 'Phrase'},
+                                              {uid: {'Resolution': 'ExifMetadata Device Manufacturer',
+                                                     'Notes': ''}}])
+                    elif tag == "model":
+                        return_result.append([{'Phrase': my_image.model,
+                                               'Entity Type': 'Phrase'},
+                                              {index_of_child: {'Resolution': 'ExifMetadata Device Model',
+                                                                'Notes': ''}}])
         return return_result

Core/Resolutions/Core/ImageToGeoLocation.py

@@ -3,7 +3,8 @@
 
 class ImageToGeoLocation:
     name = "Geolocation From Image"
-    description = "Find information about where the image was taken"
+    category = "File Operations"
+    description = "Find information about where an image was taken."
     originTypes = {"Image"}
     resultTypes = {'GeoCoordinates'}
     parameters = {}
@@ -15,19 +16,18 @@ class ImageToGeoLocation:
         return_result = []
         for entity in entityJsonList:
             uid = entity['uid']
-            image_path = Path(entity["File Path"])
-            if not (image_path.exists() and image_path.is_file()):
+            image_path = Path(parameters['Project Files Directory']) / entity['File Path']
+            if not image_path.exists() or not image_path.is_file():
                 continue
             with open(image_path, 'rb') as image_file:
                 my_image = Image(image_file)
                 if my_image.has_exif is False:
                     continue
-                else:
-                    for tag in my_image.list_all():
-                        if tag == "gps_latitude":
-                            return_result.append([{'Label': "Location of"+str(entity[list(entity)[1]].strip()),
-                                                   'Latitude': my_image.gps_latitude,
-                                                   'Longitude': my_image.gps_longitude,
-                                                   'Entity Type': 'GeoCoordinates'},
-                                                  {uid: {'Resolution': 'GeoCoordinates', 'Notes': ''}}])
+                return_result.extend([{'Label': f"Location of {str(entity[list(entity)[1]].strip())}",
+                                       'Latitude': my_image.gps_latitude,
+                                       'Longitude': my_image.gps_longitude,
+                                       'Entity Type': 'GeoCoordinates'},
+                                      {uid: {'Resolution': 'GeoCoordinates', 'Notes': ''}}]
+                                     for tag in my_image.list_all() if tag == "gps_latitude")
+
         return return_result

Core/Resolutions/Core/JSCodeExtractor.py

@@ -5,6 +5,8 @@ class JSCodeExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Tracking Codes"
 
+    category = "Website Tracking"
+
     # A string that describes this resolution.
     description = "Returns Nodes of 'ca-pub', 'ua' and 'gtm' tracking codes for websites and/or domains."
 
@@ -15,65 +17,196 @@ class JSCodeExtractor:
     parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright
+        from playwright.sync_api import sync_playwright, Error
+        from base64 import b64decode
+        from pathlib import Path
         import re
+        import contextlib
+
+        playwrightPath = Path(parameters['Playwright Firefox'])
         returnResults = []
         requestUrlsParsed = set()
 
         uaRegex = re.compile(r'\bUA-\d{4,10}-\d{1,4}\b', re.IGNORECASE)
         pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
-        gtmRegex = re.compile(r'\bGTM-[A-Z0-9]{1,7}\b', re.IGNORECASE)
-        gRegex = re.compile(r'\bG-[A-Z0-9]{1,15}\b', re.IGNORECASE)
-        qualtricsRegex = re.compile(r'\bQ_ZID=[a-zA-Z_0-9]*\b', re.IGNORECASE)
+        gtmRegex = re.compile(r'\bGTM-[A-Z\d]{1,7}\b')
+        gRegex = re.compile(r'\bG-[A-Z\d]{1,15}\b', re.IGNORECASE)
+        qualtricsRegex = re.compile(r'\bQ_[ZS]ID=\w*\b')
+        pingdomRegex = re.compile(r'\bpa-[a-fA-F\d]{24}.js\b$')
+        mPulseRegex = re.compile(r'go-mpulse.net/boomerang/[A-Z\d]{5}(?:-[A-Z\d]{5}){4}\b')
+        contextWebRegex = re.compile(r'\.contextweb\.com.*token=.*')
+        facebookRegex = re.compile(r'facebook.com/tr?.*id=\d*')
+        googleMapsRegex = re.compile(r'maps\.googleapis\.com/maps/api/js\?.*client=[ a-zA-Z\d-]*')
+        marketoRegex = re.compile(r'marketo\.com/rtp-api/v1/rtp.js\?.*aid=[ a-zA-Z\d-]*')
+        visualWebsiteOptimizerRegex = re.compile(r'visualwebsiteoptimizer\.com/j\.php\?.*a=\d*')
+        optimizeRegex = re.compile(r'googleoptimize\.com/optimize\.js\?.*id=[A-Z\d-]*')
+        markMonitorRegex = re.compile(r'\.adsrvr\.org/track/evnt/\?.*adv=[a-zA-Z\d-]*')
+        zendeskRegex = re.compile(r'\.zdassets\.com/ekr/snippet\.js\?.*key=[a-zA-Z\d-]*')
+        quantServeRegex = re.compile(r'pixel\.quantserve\.com/pixel/.*\.gif\?')
+        cookieLawRegex = re.compile(r'cdn\.cookielaw\.org/consent/.*/')
+        oneTagRegex = re.compile(r'get\.s-onetag\.com/.*/')
+        bounceExchangeRegex = re.compile(r'tag\.bounceexchange\.com/.*/')
+        pushlyRegex = re.compile(r'cdn.p-n.io/.*domain_key=[\w%]*')
+        akamaiRegex = re.compile(r'/akam/.*a=[\w%=]*')
+        demdexRegex = re.compile(r'dpm\.demdex\.net/id\?.*d_orgid=[^&]*')
+        brightcoveRegex = re.compile(r'metrics\.brightcove\.com/.*/tracker\?.*&account=[^&]*')
 
         def GetTrackingCodes(pageUid, requestUrl) -> None:
-            if requestUrl not in requestUrlsParsed:
-                requestUrlsParsed.add(requestUrl)
-                for uaCode in uaRegex.findall(requestUrl):
-                    returnResults.append([{'Phrase': uaCode,
-                                           'Entity Type': 'Phrase'},
-                                          {pageUid: {'Resolution': 'Google UA Tracking Code',
-                                                     'Notes': ''}}])
-                for pubCode in pubRegex.findall(requestUrl):
-                    returnResults.append([{'Phrase': pubCode,
-                                           'Entity Type': 'Phrase'},
-                                          {pageUid: {'Resolution': 'Google AdSense ca-pub Tracking Code',
-                                                     'Notes': ''}}])
-                for gtmCode in gtmRegex.findall(requestUrl):
-                    returnResults.append([{'Phrase': gtmCode,
-                                           'Entity Type': 'Phrase'},
-                                          {pageUid: {'Resolution': 'Google GTM Tracking Code',
-                                                     'Notes': ''}}])
-                for gCode in gRegex.findall(requestUrl):
-                    returnResults.append([{'Phrase': gCode,
-                                           'Entity Type': 'Phrase'},
-                                          {pageUid: {'Resolution': 'Google G Tracking Code',
-                                                     'Notes': ''}}])
-                for qCode in qualtricsRegex.findall(requestUrl):
-                    returnResults.append([{'Phrase': qCode[6:],
-                                           'Entity Type': 'Phrase'},
-                                          {pageUid: {'Resolution': 'Qualtrics Tracking Code',
-                                                     'Notes': ''}}])
+            if requestUrl in requestUrlsParsed:
+                return
+            requestUrlsParsed.add(requestUrl)
+            for uaCode in uaRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': uaCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google UA Tracking Code',
+                                                 'Notes': ''}}])
+            for pubCode in pubRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pubCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google AdSense ca-pub Tracking Code',
+                                                 'Notes': ''}}])
+            for gtmCode in gtmRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': gtmCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google GTM Tracking Code',
+                                                 'Notes': ''}}])
+            for gCode in gRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': gCode,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google G Tracking Code',
+                                                 'Notes': ''}}])
+            for qCode in qualtricsRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': qCode[6:],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Qualtrics Tracking Code',
+                                                 'Notes': ''}}])
+            for pCode in pingdomRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pCode[:-3],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Pingdom Tracking Code',
+                                                 'Notes': ''}}])
+            for mCode in mPulseRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mCode.split('/')[-1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'mPulse Tracking Code',
+                                                 'Notes': ''}}])
+            for cCode in contextWebRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': cCode.split('token=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'ContextWeb Tracking Code',
+                                                 'Notes': ''}}])
+            for fCode in facebookRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': fCode.split('id=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Facebook Tracking Pixel Code',
+                                                 'Notes': ''}}])
+            for mapsCode in googleMapsRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mapsCode.split('client=', 1)[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google Maps Client Code',
+                                                 'Notes': ''}}])
+            for marketoCode in marketoRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': marketoCode.split('aid=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Marketo Tracking Code',
+                                                 'Notes': ''}}])
+            for vwoCode in visualWebsiteOptimizerRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': vwoCode.split('a=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Visual Website Optimizer Tracking User ID',
+                                                 'Notes': ''}}])
+            for oCode in optimizeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': oCode.split('id=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Google Optimize ID',
+                                                 'Notes': ''}}])
+            for mmCode in markMonitorRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': mmCode.split('adv=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Mark Monitor Tracking ID',
+                                                 'Notes': ''}}])
+            for zCode in zendeskRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': zCode.split('key=')[1].split('&')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Zendesk ID',
+                                                 'Notes': ''}}])
+            for qsCode in quantServeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': qsCode.split('/pixel/')[1].split('.gif')[0],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'QuantServe Tracking Pixel ID',
+                                                 'Notes': ''}}])
+            for clCode in cookieLawRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': clCode.split('/')[2],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'CookieLaw Website ID',
+                                                 'Notes': ''}}])
+            for otCode in oneTagRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': otCode.split('/')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'OneTag Tracking ID',
+                                                 'Notes': ''}}])
+            for beCode in bounceExchangeRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': beCode.split('/')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'BounceExchange Tracking ID',
+                                                 'Notes': ''}}])
+            for pushlyCode in pushlyRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': pushlyCode.split('domain_key=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Pushly Website ID',
+                                                 'Notes': ''}}])
+            for aCode in akamaiRegex.findall(requestUrl):
+                encodedTracking = aCode.split('a=', 1)[1]
+                encodedTracking = encodedTracking.replace('%3D', '=')
+                decodedTracking = b64decode(encodedTracking).decode('utf-8').split('t=')[1].split('&')[0]
+                returnResults.append([{'Phrase': decodedTracking,
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'Akamai Website ID',
+                                                 'Notes': 'SHA-1 Sum'}}])
+            for dCode in demdexRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': dCode.split('d_orgid=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'DemDex (Adobe) Website ID',
+                                                 'Notes': ''}}])
+            for bCode in brightcoveRegex.findall(requestUrl):
+                returnResults.append([{'Phrase': bCode.split('account=')[1],
+                                       'Entity Type': 'Phrase'},
+                                      {pageUid: {'Resolution': 'BrightCove Website ID',
+                                                 'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'
             )
-            page = context.new_page()
+            contextNoJS = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0',
+                java_script_enabled=False
+            )
+            pageJS = context.new_page()
+            pageNoJS = contextNoJS.new_page()
+            uid = None
+
+            # Subscribe to "request" events.
+            pageJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+            pageNoJS.on("request", lambda request: GetTrackingCodes(uid, request.url))
+
             for site in entityJsonList:
                 uid = site['uid']
                 url = site.get('URL') if site.get('Entity Type', '') == 'Website' else site.get('Domain Name', None)
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
+                    url = f'http://{url}'
 
-                # Subscribe to "request" events.
-                page.on("request", lambda request: GetTrackingCodes(uid, request.url))
-                page.goto(url)
-            page.close()
+                with contextlib.suppress(Error):
+                    pageJS.goto(url, wait_until="networkidle")
+                with contextlib.suppress(Error):
+                    pageNoJS.goto(url, wait_until="networkidle")
+            pageJS.close()
+            pageNoJS.close()
             browser.close()
 
         return returnResults

Core/Resolutions/Core/LongANStringExtractor.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+"""
+Credit to @cyb_detective:
+https://medium.com/@cyb_detective/20-regular-expressions-examples-to-search-for-data-related-to-cryptocurrencies-43e31dd4a5dc
+"""
+
+
+class LongANStringExtractor:
+    # A string that is treated as the name of this resolution.
+    name = "Extract Long Alphanumeric Strings"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Returns patterns matching common cryptocurrency address formats on a website."
+
+    originTypes = {'Domain', 'Website'}
+
+    resultTypes = {'Phrase'}
+
+    parameters = {'Minimum Length': {'description': 'Specify the minimum length an alphanumeric string has to have '
+                                                    'to be extracted.',
+                                     'type': 'String',
+                                     'value': '',
+                                     'default': '20'
+                                     }}
+
+    def resolution(self, entityJsonList, parameters):
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
+        from bs4 import BeautifulSoup
+        from pathlib import Path
+        import re
+
+        playwrightPath = Path(parameters['Playwright Chromium'])
+        try:
+            minLength = int(parameters['Minimum Length'])
+            if minLength < 1:
+                raise ValueError('Invalid min length specified.')
+        except ValueError:
+            return "Invalid Minimum Length specified."
+
+        returnResults = []
+
+        matchPattern = re.compile(r"\b[a-zA-Z0-9_.-]{" + str(minLength) + r",}\b")
+
+        # The software can deduplicate, but handling it here is better.
+        allPatterns = set()
+
+        def extractStrings(currentUID: str, site: str):
+            page = context.new_page()
+            pageResolved = False
+            for _ in range(3):
+                try:
+                    page.goto(site, wait_until="networkidle", timeout=10000)
+                    pageResolved = True
+                    break
+                except TimeoutError:
+                    pass
+                except Error:
+                    break
+            if not pageResolved:
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
+
+            soupContents = BeautifulSoup(page.content(), 'lxml')
+            # Remove <span> and <noscript> tags.
+            while True:
+                try:
+                    soupContents.noscript.extract()
+                except AttributeError:
+                    break
+            while True:
+                try:
+                    soupContents.span.extract()
+                except AttributeError:
+                    break
+            siteContent = soupContents.get_text()
+
+            stringMatches = matchPattern.findall(siteContent)
+            for potentialMatch in stringMatches:
+                if potentialMatch not in allPatterns:
+                    allPatterns.add(potentialMatch)
+                    returnResults.append([{'Phrase': potentialMatch,
+                                           'Entity Type': 'Phrase'},
+                                          {currentUID: {'Resolution': 'Long alphanumeric string',
+                                                        'Notes': ''}}])
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(executable_path=playwrightPath)
+            context = browser.new_context(
+                viewport={'width': 1920, 'height': 1080},
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36'
+            )
+            for entity in entityJsonList:
+                uid = entity['uid']
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
+                if url is None:
+                    continue
+                if not url.startswith('http://') and not url.startswith('https://'):
+                    url = f'http://{url}'
+                extractStrings(uid, url)
+            browser.close()
+
+        return returnResults

Core/Resolutions/Core/NPMJSSearch.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+
+class NPMJSSearch:
+    name = "Find NPM organization"
+    category = "Online Identity"
+    description = "Find a collective's npmjs organization page."
+    originTypes = {'Phrase', 'Company', 'Organization'}
+    resultTypes = {'Website'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import requests
+
+        headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0'}
+        url_base = 'https://www.npmjs.com/org/'
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity[list(entity)[1]].lower()
+
+            string_checks = {''.join(primaryField.split(' '))}
+            string_checks.add('_'.join(primaryField.split(' ')))
+            string_checks.add('-'.join(primaryField.split(' ')))
+
+            for check in string_checks:
+                check_url = url_base + check
+                request = requests.head(check_url, headers=headers)
+                if request.status_code == 200:
+                    returnResults.append([{'URL': check_url,
+                                           'Entity Type': 'Website'},
+                                          {entity['uid']: {'Resolution': 'NPMJS Org',
+                                                           'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/PhoneNumbersExtractor.py

@@ -10,6 +10,8 @@ class PhoneNumbersExtractor:
     # A string that is treated as the name of this resolution.
     name = "Extract Phone Numbers"
 
+    category = "Website Information"
+
     # A string that describes this resolution.
     description = "Returns the Phone Numbers present on a website or index page of a domain."
 
@@ -17,34 +19,22 @@ class PhoneNumbersExtractor:
 
     resultTypes = {'Phone Number'}
 
-    parameters = {'Max Depth': {'description': 'Each link leading to another website in the same domain can be '
-                                               'explored to discover more entities. Each entity discovered after '
-                                               'exploring sites linked in the original website or domain is said to '
-                                               'have a "depth" value of 1. Entities found from exploring the links on '
-                                               'this page would have a "depth" of 2, and so on. A larger value could '
-                                               'result in EXPONENTIALLY more time taken to finish the resolution.\n'
-                                               'The default value is "0", which means only the provided website, or '
-                                               'the index page of the domain provided, is explored.',
-                                'type': 'String',
-                                'value': '0',
-                                'default': '0'}}
+    parameters = {}
 
     def resolution(self, entityJsonList, parameters):
-        from playwright.sync_api import sync_playwright, TimeoutError
+        from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
-        import tldextract
+        from pathlib import Path
+        import re
+
+        playwrightPath = Path(parameters['Playwright Chromium'])
+
+        cleanTagsRegex = re.compile(r'<.*?>')
+        phoneNumCharsExclusion = re.compile(r'[^ -+()\[\]\d]')
 
         returnResults = []
 
-        # Numbers less than zero are the same as zero, but we should try to prevent overflows.
-        try:
-            maxDepth = max(int(parameters['Max Depth']), 0)
-        except ValueError:
-            return "Invalid value provided for Max Webpages to follow."
-
-        exploredDepth = set()
-
-        def extractTels(currentUID: str, site: str, depth: int):
+        def extractTels(currentUID: str, site: str):
             page = context.new_page()
             pageResolved = False
             for _ in range(3):
@@ -54,53 +44,54 @@ class PhoneNumbersExtractor:
                     break
                 except TimeoutError:
                     pass
+                except Error:
+                    break
             if not pageResolved:
-                return
+                # Last chance for this to work; some pages have issues with the "networkidle" trigger.
+                try:
+                    page.goto(site, wait_until="load", timeout=10000)
+                except Error:
+                    return
 
             soupContents = BeautifulSoup(page.content(), 'lxml')
 
             linksInAHref = soupContents.find_all('a')
             for tag in linksInAHref:
                 newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('tel:'):
-                        returnResults.append([{'Phone Number': newLink[4:],
+                if newLink is not None and newLink.startswith('tel:'):
+                    returnResults.append([{'Phone Number': newLink[4:],
+                                           'Entity Type': 'Phone Number'},
+                                          {currentUID: {'Resolution': 'Phone Number Found',
+                                                        'Notes': ''}}])
+
+            textTags = soupContents.find_all('p')
+            for tag in textTags:
+                tagContents = tag.text
+                cleanTagContentsList = re.sub(cleanTagsRegex, '', tagContents).split('\n')
+                for cleanTag in cleanTagContentsList:
+                    cleanTag = cleanTag.strip()
+                    if not phoneNumCharsExclusion.findall(cleanTag) and len(re.findall(r'\d', cleanTag)) >= 3:
+                        returnResults.append([{'Phone Number': cleanTag,
                                                'Entity Type': 'Phone Number'},
                                               {currentUID: {'Resolution': 'Phone Number Found',
                                                             'Notes': ''}}])
-                    elif newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractTels(currentUID, newLink, newDepth)
-
-            linksInLinkHref = soupContents.find_all('link')
-            for tag in linksInLinkHref:
-                newLink = tag.get('href', None)
-                if newLink is not None:
-                    if newLink.startswith('http'):
-                        newLink = newLink.split('#')[0]
-                        newDepth = depth - 1
-                        if domain in newLink and newLink not in exploredDepth and newDepth > 0:
-                            exploredDepth.add(newLink)
-                            extractTels(currentUID, newLink, newDepth)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
+                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                           'Chrome/101.0.4951.54 Safari/537.36',
             )
             for entity in entityJsonList:
                 uid = entity['uid']
-                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else entity.get('Domain Name', None)
+                url = entity.get('URL') if entity.get('Entity Type', '') == 'Website' else \
+                    entity.get('Domain Name', None)
                 if url is None:
                     continue
                 if not url.startswith('http://') and not url.startswith('https://'):
-                    url = 'http://' + url
-                domain = tldextract.extract(url).fqdn
-                extractTels(uid, url, maxDepth)
+                    url = f'http://{url}'
+                extractTels(uid, url)
             browser.close()
 
         return returnResults

Core/Resolutions/Core/PhraseSimilarity.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
 class PhraseSimilarity:
-    name = "Perform Similarity Check"
-    description = "Find the ration of similarity between two strings"
+    name = "String Similarity Check"
+    category = "String Operations"
+    description = "Find the ratio of similarity between two strings."
     originTypes = {'*'}
     resultTypes = {'Phrase'}
     parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
@@ -11,8 +12,7 @@ class PhraseSimilarity:
                   'Algorithm': {'description': 'Select the Algorithm to use',
                                 'type': 'SingleChoice',
                                 'value': {'levenshtein distance', 'damerau levenshtein distance',
-                                          'jaro distance', 'jaro winkler similarity',
-                                          'match rating comparison', 'hamming distance'}}
+                                          'jaro distance', 'jaro winkler similarity', 'hamming distance'}}
                   }
 
     def resolution(self, entityJsonList, parameters):
@@ -20,33 +20,28 @@ class PhraseSimilarity:
         from itertools import combinations
 
         return_result = []
-        primary_fields = []
-        notes_fields = []
-        uidList = []
+        entity_fields = []
         selection = parameters['Primary field or Notes']
         algorithm = parameters['Algorithm'].replace(" ", "_")
         for entity in entityJsonList:
-            uidList.append(entity['uid'])
             if selection == 'Primary Field':
-                primary_fields.append(entity[list(entity)[1]].strip())
+                entity_fields.append((entity['uid'], entity[list(entity)[1]].strip()))
             elif selection == 'Notes':
-                notes_fields.append(entity['Notes'])
+                entity_fields.append((entity['uid'], entity.get('Notes')))
 
-        if selection == "Notes" and len(notes_fields) > 1:
-            value = list(combinations(notes_fields, 2))
-            uid = list(combinations(uidList, 2))
-        elif selection == "Primary Field" and len(primary_fields) > 1:
-            value = list(combinations(primary_fields, 2))
-            uid = list(combinations(uidList, 2))
-        else:
+        if len(entity_fields) < 2:
             return "Please Select 2 or more entities for comparison"
-        for field in value:
-            similarity = getattr(jellyfish, algorithm)(field[0], field[1])
-            return_result.append([{'Phrase': f"{similarity} similarity",
+        for combination in combinations(entity_fields, 2):
+            stringA = combination[0][1]
+            uidA = combination[0][0]
+            stringB = combination[1][1]
+            uidB = combination[1][0]
+            similarity = getattr(jellyfish, algorithm)(stringA, stringB)
+            return_result.append([{'Phrase': f'{algorithm} between "{stringA}" and "{stringB}": {similarity}',
                                    'Entity Type': 'Phrase'},
-                                  {uid[value.index(field)][0]: {'Resolution': 'Notes Word Counter', 'Notes': ''}}])
-            return_result.append([{'Phrase': f"{similarity} similarity",
-                                   'Entity Type': 'Phrase'},
-                                  {uid[value.index(field)][1]: {'Resolution': 'Notes Word Counter', 'Notes': ''}}])
+                                  {uidA: {'Resolution': f'String Similarity for "{stringA}" and "{stringB}"',
+                                          'Notes': ''},
+                                   uidB: {'Resolution': f'String Similarity for "{stringA}" and "{stringB}"',
+                                          'Notes': ''}}])
 
         return return_result

Core/Resolutions/Core/RegexMatch.py

@@ -5,23 +5,23 @@ class RegexMatch:
     # A string that is treated as the name of this resolution.
     name = "Get Regex Match"
 
+    category = "String Operations"
+
     # A string that describes this resolution.
-    description = "Returns Nodes of contact info for websites"
+    description = "Extract text from Phrases and Websites with Regular Expressions."
 
     originTypes = {'Phrase', 'Website'}
 
     resultTypes = {'Phrase'}
 
-    parameters = {'Regex Match': {'description': "Please enter the Regex to be searched for.\n"
-                                                 "if any matches are found to be exactly the same as the entity's "
-                                                 "primary field they will be ignored",
+    parameters = {'Regex Match': {'description': "Please enter the Regex expression to extract strings with.",
                                   'type': 'String',
                                   'value': ''},
-                  'Max Results': {'description': 'Please enter the Maximum number of Results to return',
+                  'Max Results': {'description': 'Please enter the Maximum number of Results to return.',
                                   'type': 'String',
                                   'value': '',
                                   'default': '5'},
-                  'Re Flags': {'description': 'Select Re Flags to be used while compiling the regex',
+                  'Re Flags': {'description': 'Select the Regex Flags to be used.',
                                'type': 'MultiChoice',
                                'value': {'re.I', 're.M', 're.S'}
                                }
@@ -32,46 +32,36 @@ class RegexMatch:
         import requests
 
         headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
         }
 
         returnResults = []
         try:
-            linkNumbers = int(parameters['Max Results'])
+            maxResults = int(parameters['Max Results'])
         except ValueError:
             return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
+        if maxResults <= 0:
             return []
         search_param = parameters['Regex Match']
         flags = parameters['Re Flags']
-        print(flags)
+
+        flagsToUse = 0
+        for flag in flags:
+            flagsToUse |= flag
 
         for entity in entityJsonList:
             uid = entity['uid']
             if entity['Entity Type'] == 'Phrase':
-                text = str(entity['Notes']) + str(entity['Phrase'])
+                text = str(entity['Notes']) + "\n" + str(entity['Phrase'])
             else:  # Website entity
                 r = requests.get(entity["URL"], headers=headers)
                 text = r.text
 
-            if len(flags) == 3:
-                search_re = re.findall(search_param, text, flags=re.I | re.S | re.M)
-            elif len(flags) == 2:
-                search_re = re.findall(search_param, text, flags=flags[0] | flags[1])
-            elif len(flags) == 1:
-                search_re = re.findall(search_param, text, flags=flags[0])
-            else:
-                search_re = re.findall(search_param, text)
+            search_re = re.findall(search_param, text, flags=flagsToUse)
 
-            if linkNumbers > len(search_re):
-                linkNumbers = int(len(search_re))
-
-            for i in range(linkNumbers):
-                if search_re[i] == entity[list(entity)[1]]:
-                    continue
-                returnResults.append([{'Phrase': search_re[i],
-                                       'Entity Type': 'Phrase',
-                                       'Notes': ''},
-                                      {uid: {'Resolution': 'Phrase Details',
-                                             'Notes': ''}}])
+            returnResults.extend([{'Phrase': f'Regex Match: {regexMatch}',
+                                   'Entity Type': 'Phrase',
+                                   'Notes': ''},
+                                  {uid: {'Resolution': 'Regex String Match', 'Notes': ''}}]
+                                 for regexMatch in search_re[:maxResults])
         return returnResults

Core/Resolutions/Core/ReplacePhrase.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+
+class ReplacePhrase:
+    name = "Replace String in Phrase"
+    category = "String Operations"
+    description = "Replace a character, or sequence of characters in a Phrase with another character or sequence."
+    originTypes = {'Phrase'}
+    resultTypes = {'Phrase'}
+
+    parameters = {'Sequence to Remove': {'description': 'Specify the character or sequence of characters to replace.',
+                                         'type': 'String',
+                                         'value': ''
+                                         },
+                  'Sequence to Insert': {'description': 'Specify the character or sequence of characters to replace '
+                                                        'the old character or sequence with. Enter the same character '
+                                                        'or sequence to delete the character or sequence instead.',
+                                         'type': 'String',
+                                         'value': ''
+                                         },
+                  'Match Type': {'description': 'Specify whether the matching of characters to replace '
+                                                'should be plain (as in, match characters as they were typed), '
+                                                'case insensitive, or regex.',
+                                 'type': 'SingleChoice',
+                                 'value': {'Plain', 'Case Insensitive', 'Regex'},
+                                 'default': 'Plain'
+                                 },
+                  'Match Count': {'description': 'Specify the number of times to replace the specified character or '
+                                                 'sequence with the new sequence. Zero is unlimited times.',
+                                  'type': 'String',
+                                  'value': '0',
+                                  'default': '0'
+                                  }
+                  }
+
+    def resolution(self, entityJsonList, parameters):
+        import re
+
+        returnResults = []
+
+        remove = parameters['Sequence to Remove']
+        insert = parameters['Sequence to Insert']
+        if insert == remove:
+            insert = ''
+        matchType = parameters['Match Type']
+
+        try:
+            matchCount = int(parameters['Match Count'])
+            if matchCount < 0:
+                return []
+        except ValueError:
+            return "Invalid Match Count specified."
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase']
+
+            if matchType == 'Plain':
+                if matchCount == 0:
+                    matchCount = -1
+                primaryField = primaryField.replace(remove, insert, matchCount)
+            elif matchType == 'Case Insensitive':
+                pattern = re.compile(re.escape(remove), re.IGNORECASE)
+                primaryField = pattern.sub(insert, primaryField, matchCount)
+            else:
+                pattern = re.compile(remove)
+                primaryField = pattern.sub(insert, primaryField, matchCount)
+
+            returnResults.append([{'Phrase': primaryField,
+                                   'Entity Type': 'Phrase'},
+                                  {entity['uid']: {'Resolution': 'Replace characters',
+                                                   'Notes': ''}}])
+
+        return returnResults

Core/Resolutions/Core/TikTokVideoPublishDetails.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+class TikTokVideoPublishDetails:
+    # A string that is treated as the name of this resolution.
+    name = "TikTok Video Publishing Details"
+
+    category = "Website Information"
+
+    # A string that describes this resolution.
+    description = "Extracts information from public TikTok Videos."
+
+    originTypes = {'Website'}
+
+    resultTypes = {'Social Media Handle', 'Date'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        from datetime import datetime
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            uid = entity['uid']
+            url = entity['URL'].lower()
+
+            try:
+                splitURL = url.split('/')
+                username = splitURL[3]
+                videoID = int(splitURL[5].split('?')[0])
+            except (IndexError, ValueError):
+                continue
+
+            binString = "{0:b}".format(videoID)
+            if len(binString) == 63:
+                binString = f'0{binString}'
+            binString = int(binString[:32], 2)
+
+            UTCTimestamp = f'{datetime.utcfromtimestamp(binString).isoformat()}+00:00'
+
+            returnResults.extend(([{'Date': UTCTimestamp, 'Entity Type': 'Date'},
+                                   {uid: {'Resolution': 'Video Publish Date', 'Notes': ''}}],
+                                  [{'User Name': username, 'Entity Type': 'Social Media Handle'},
+                                   {uid: {'Resolution': 'Published By', 'Notes': ''}}]))
+
+        return returnResults

Core/Resolutions/Core/ToPhrase.py

@@ -3,7 +3,8 @@
 
 class ToPhrase:
     name = "Convert To Phrase"
-    description = "Convert the entity in question to a Phrase entity. Uses only the primary field."
+    category = "String Operations"
+    description = "Convert the primary field of selected entities to a Phrase entity."
     originTypes = {'*'}
     resultTypes = {'Phrase'}
 

Core/Resolutions/Core/WebsiteFromPhrase.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+
+class WebsiteFromPhrase:
+    name = "Website From Phrase"
+    category = "String Operations"
+    description = "Extract website-like objects from a phrase."
+    originTypes = {'Phrase'}
+    resultTypes = {'Domain'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        import re
+        import tldextract
+
+        websiteRegex = re.compile(r"""^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$""")
+        wordChar = re.compile(r'\w')
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['Phrase']
+            for entityChunk in primaryField.split():
+                while wordChar.match(entityChunk[-1]) is None:
+                    entityChunk = entityChunk[:-1]
+                if websiteRegex.match(entityChunk):
+                    with contextlib.suppress(Exception):
+                        tldObject = tldextract.extract(entityChunk)
+                        if tldObject.suffix != '':
+                            returnResults.append([{'URL': entityChunk,
+                                                   'Entity Type': 'Website'},
+                                                  {entity['uid']: {'Resolution': 'Phrase To Website',
+                                                                   'Notes': ''}}])
+        return returnResults

Core/Resolutions/Core/WordCounter.py

@@ -3,12 +3,14 @@
 
 class WordCounter:
     name = "Get Number Of Words"
-    description = "Find the number of words in a given phrase"
+    category = "String Operations"
+    description = "Count the number of words in a given phrase."
     originTypes = {'Phrase'}
     resultTypes = {'Phrase'}
     parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
                                              'type': 'SingleChoice',
-                                             'value': {'Notes', 'Primary Field'}}}
+                                             'value': {'Notes', 'Primary Field'},
+                                             'default': 'Notes'}}
 
     def resolution(self, entityJsonList, parameters):
         import re

Core/ResourceHandler.py

@@ -1,6 +1,13 @@
 #!/usr/bin/env python3
 
+
+import contextlib
+import re
+from typing import Union, Optional
+from glob import glob
+
 import networkx as nx
+from datetime import timezone
 from defusedxml.ElementTree import parse
 from datetime import datetime
 from os import listdir
@@ -8,9 +15,44 @@ from pathlib import Path
 from uuid import uuid4
 from ast import literal_eval
 from base64 import b64decode
+from dateutil import parser
 
-from PySide6.QtGui import QIcon
-from PySide6.QtCore import QByteArray
+from PIL import Image
+from PIL.ImageQt import ImageQt
+from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize, QUrl, Qt
+from PySide6 import QtWidgets, QtGui
+
+
+def resizePictureFromBuffer(picBuffer: QByteArray, newSize: tuple) -> QByteArray:
+    """
+    newSize: First is width, second is height.
+    """
+    picBufferData = picBuffer.data()
+    if picBufferData.startswith(b'<svg ') or picBufferData.startswith(b'<?xml '):
+        return QByteArray(resizeSVG(picBufferData, newSize))
+    originalImage = QtGui.QImage()
+    originalImage.loadFromData(picBuffer)
+    newImage = originalImage.scaled(newSize[0], newSize[1])
+
+    pictureByteArray = QByteArray()
+    imageBuffer = QBuffer(pictureByteArray)
+    imageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+    newImage.save(imageBuffer, "PNG")
+    imageBuffer.close()
+
+    return pictureByteArray
+
+
+def resizeSVG(byteString: bytes, resize: tuple):
+    bytesWidth = str(resize[0]).encode('UTF-8')
+    bytesHeight = str(resize[1]).encode('UTF-8')
+    widthRegex = re.compile(b' width="\d*" ')
+    for widthMatches in widthRegex.findall(byteString):
+        byteString = byteString.replace(widthMatches, b' ')
+    heightRegex = re.compile(b' height="\d*" ')
+    for heightMatches in heightRegex.findall(byteString):
+        byteString = byteString.replace(heightMatches, b' ')
+    return byteString.replace(b'<svg ', b'<svg height="%b" width="%b" ' % (bytesHeight, bytesWidth), 1)
 
 
 class ResourceHandler:
@@ -19,144 +61,260 @@ class ResourceHandler:
         return self.icons[iconName]
 
     # Load all resources needed.
-    def __init__(self, mainWindow, messageHandler):
+    def __init__(self, mainWindow) -> None:
         self.mainWindow = mainWindow
-        self.messageHandler = messageHandler
+        self.programBaseDirPath = Path(self.mainWindow.SETTINGS.value("Program/BaseDir"))
         self.entityCategoryList = {}
+        self.moduleAssetPaths = []
 
-        self.icons = {"uploading": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "Uploading.png"),
-                      "uploaded": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                      "Resources" / "Icons" / "Uploaded.png"),
-                      "upArrow": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                     "Resources" / "Icons" / "UpArrow.png"),
-                      "downArrow": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "DownArrow.png"),
-                      "isolatedNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                           "Resources" / "Icons" / "SelectIsolated.png"),
-                      "addCanvas": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "Add_Canvas.png"),
-                      "generateReport": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                            "Resources" / "Icons" / "Generate_Report.png"),
-                      "leafNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "SelectLeaf.png"),
-                      "nonIsolatedNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                              "Resources" / "Icons" / "SelectNonIsolated.png"),
-                      "rootNodes": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "SelectRoot.png"),
-                      "split": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                   "Resources" / "Icons" / "Split.png"),
-                      "merge": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                   "Resources" / "Icons" / "Merge.png"),
-                      "shortestPath": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                          "Resources" / "Icons" / "ShortestPath.png"),
-                      "drawLink": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                      "Resources" / "Icons" / "DrawLink.png"),
-                      "rearrange": str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) /
-                                       "Resources" / "Icons" / "RearrangeGraph.png"),
+        self.icons = {"uploading": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploading.png"),
+                      "uploaded": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploaded.png"),
+                      "upArrow": str(self.programBaseDirPath / "Resources" / "Icons" / "UpArrow.png"),
+                      "downArrow": str(self.programBaseDirPath / "Resources" / "Icons" / "DownArrow.png"),
+                      "isolatedNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectIsolated.png"),
+                      "addCanvas": str(self.programBaseDirPath / "Resources" / "Icons" / "Add_Canvas.png"),
+                      "generateReport": str(self.programBaseDirPath / "Resources" / "Icons" / "Generate_Report.png"),
+                      "leafNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectLeaf.png"),
+                      "nonIsolatedNodes": str(self.programBaseDirPath / "Resources" / "Icons" /
+                                              "SelectNonIsolated.png"),
+                      "rootNodes": str(self.programBaseDirPath / "Resources" / "Icons" / "SelectRoot.png"),
+                      "split": str(self.programBaseDirPath / "Resources" / "Icons" / "Split.png"),
+                      "merge": str(self.programBaseDirPath / "Resources" / "Icons" / "Merge.png"),
+                      "shortestPath": str(self.programBaseDirPath / "Resources" / "Icons" / "ShortestPath.png"),
+                      "drawLink": str(self.programBaseDirPath / "Resources" / "Icons" / "DrawLink.png"),
+                      "rearrange": str(self.programBaseDirPath / "Resources" / "Icons" / "RearrangeGraph.png"),
+                      "colorPicker": str(self.programBaseDirPath / "Resources" / "Icons" / "ColorPicker.png"),
                       }
 
-        self.loadCoreEntities()
+        self.banners = {f"{bannerPath.split('Banner_')[-1].split('.')[0]}": str(bannerPath)
+                        for bannerPath in glob(str(self.programBaseDirPath / "Resources" / "Icons" / "Banner_*.svg"))}
+        # These are not meant to be strict - just restrictive enough such that users don't put in utter nonsense.
+        # Note that regex isn't always the best way of validating fields, but it should be good enough for our
+        #   purposes.
+        self.checks = {'Email': re.compile(
+            r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)])"""),
+            'Phonenumber': re.compile(r"""^(\+|00)?[0-9() \-]{3,32}$"""),
+            'String': re.compile(r""".+"""),
+            'URL': re.compile(r"""[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)"""),
+            'Onion': re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$"""),
+            'Domain': re.compile(r"""^(\S(?<!\.)(?!/)(?<!/)){1,63}(\.(\S(?<!\.)(?!/)(?<!/)){1,63})+$"""),
+            'Float': re.compile(r"""^([-+])?(\d|\.(?=\d))+$"""),
+            'WordString': re.compile(r"""^\D+$"""),
+            'Numbers': re.compile(r"""^\d+$"""),
+            'IPv4': re.compile(r"""^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)(\.(?!$)|$)){4}$"""),
+            'IPv6': re.compile(
+                r"""^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9]))$"""),
+            'MAC': re.compile(r"""^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"""),
+            'ASN': re.compile(r"""^(AS)?\d+$"""),
+            'CUSIP': re.compile(r"""^[a-zA-Z0-9]{9}$"""),
+            'EIN': re.compile(r"""^\d{2}-?\d{7}$"""),
+            'LEIID': re.compile(r"""^[a-zA-Z0-9]{20}$"""),
+            'ISINID': re.compile(r"""^[a-zA-Z0-9]{2}-?[a-zA-Z0-9]{9}-?[a-zA-Z0-9]$"""),
+            'SIC/NAICS': re.compile(r"""^[0-9]{4,6}$""")}
 
-    def getEntityCategories(self):
-        eList = []
-        for category in self.entityCategoryList:
-            eList.append(category)
-        return eList
+        self.loadModuleEntities(self.programBaseDirPath / "Core")
 
-    def getAllEntityDetailsWithIconsInCategory(self, category):
+    def getPictureFromFile(self, filePath: Path, resize: tuple = (0, 0)) -> Optional[QByteArray]:
+        """
+        resize: tuple, first is width, second is height.
+        """
+        try:
+            with open(filePath, 'rb') as newIconFile:
+                fileContents = newIconFile.read()
+            if fileContents.startswith(b'<svg ') or fileContents.startswith(b'<?xml '):
+                if resize != (0, 0):
+                    fileContents = resizeSVG(fileContents, resize)
+                pictureByteArray = QByteArray(fileContents)
+            else:
+                image = Image.open(str(filePath))
+                if resize != (0, 0):
+                    thumbnail = ImageQt(image.resize(resize))
+                else:
+                    thumbnail = ImageQt(image)
+                pictureByteArray = QByteArray()
+                imageBuffer = QBuffer(pictureByteArray)
+
+                imageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+
+                thumbnail.save(imageBuffer, "PNG")
+                imageBuffer.close()
+        except ValueError as ve:
+            # Image type is unsupported (for ImageQt)
+            # Supported types: 1, L, P, RGB, RGBA
+            self.mainWindow.MESSAGEHANDLER.warning(f'Invalid Image selected: {str(ve)}', popUp=True)
+            pictureByteArray = None
+
+        return pictureByteArray
+
+    def loadModuleBanners(self, modulePath: Path):
+        assetsPath = modulePath / "Assets"
+        self.banners.update({f"{bannerPath.split('Banner_')[-1].split('.')[0]}": str(bannerPath)
+                             for bannerPath in glob(str(assetsPath / "Banner_*.svg"))})
+
+    def getEntityCategories(self) -> list:
+        return list(self.entityCategoryList)
+
+    def getAllEntityDetailsWithIconsInCategory(self, category) -> list:
         eList = []
         for entity in self.entityCategoryList[category]:
             entityValue = self.entityCategoryList[category][entity]
             eList.append((self.getBareBonesEntityJson(entity),
-                          entityValue['Icon']
-                          ))
+                          entityValue['Icon']))
         return eList
 
-    def getEntityAttributes(self, entityType):
+    def getEntityAttributes(self, entityType) -> Union[None, list]:
         aList = []
         try:
             for category in self.entityCategoryList:
                 if entityType in self.entityCategoryList[category]:
-                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
-                        aList.append(attribute)
+                    aList.extend(iter(self.entityCategoryList[category][entityType]['Attributes']))
                     break
         except KeyError:
-            self.messageHandler.error("Attempted to get attributes for "
-                                      "nonexistent entity type: " + str(entityType), True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for nonexistent entity type: {entityType}", True)
             return None
         return aList
 
-    def getAllEntitiesInCategory(self, category):
+    def getAllEntitiesInCategory(self, category) -> list:
         """
         Get all Entity Types in the specified category.
         """
-        eList = []
-        for entity in self.entityCategoryList[category]:
-            eList.append(entity)
-        return eList
+        return list(self.entityCategoryList[category])
 
-    def getAllEntities(self):
+    def getCategoryOfEntityType(self, entityType: Union[str, None]):
+        return next((category for category in self.entityCategoryList
+                     if entityType in self.entityCategoryList[category]),
+                    None)
+
+    def getAllEntities(self) -> list:
         """
         Get all recognised Entity Types.
         """
         eList = []
         for category in self.getEntityCategories():
-            for entity in self.getAllEntitiesInCategory(category):
-                eList.append(entity)
+            eList.extend(iter(self.getAllEntitiesInCategory(category)))
         return eList
 
-    def addRecognisedEntityTypes(self, entityFile) -> bool:
+    def validateAttributesOfEntity(self, entityJSON: dict) -> (bool, str):
+        try:
+            entityType = entityJSON['Entity Type']
+            entityCategory = self.getCategoryOfEntityType(entityType)
+            # Attributes that become part of the entity after merging are not checked.
+            # This is fine, because resolutions (by default) don't assume that any extra fields will be present.
+            entityBaseAttributes = self.getEntityAttributes(entityType)
+            if entityCategory is not None:
+                for attribute in self.entityCategoryList[entityCategory][entityType]['Attributes']:
+                    if attribute in entityBaseAttributes:
+                        attrValue = entityJSON.get(attribute)
+                        if attrValue is None or not self.runCheckOnAttribute(
+                                attrValue,
+                                self.entityCategoryList[entityCategory][entityType]['Attributes'][attribute][1]):
+                            return f'Bad value: {str(attrValue)}'
+        except Exception:
+            return False
+        return True
+
+    def runCheckOnAttribute(self, attribute: str, check: str) -> bool:
+        """
+        Check that the attribute value given matches the regex of the category 'check'.
+        """
+        # Ignore checks - this is used for non-string attributes in special entities.
+        if check == 'None':
+            return True
+        attrCheck = self.checks.get(check)
+        if attrCheck is None:
+            return False
+        result = attrCheck.findall(attribute)
+        return len(result) == 1
+
+    def getIconPathForIconFile(self, iconFile: str) -> Union[None, Path]:
+        iconPath = self.programBaseDirPath / "Resources" / "Icons" / iconFile
+        if iconPath.exists():
+            return iconPath
+        for assetPath in self.moduleAssetPaths:
+            iconPath = assetPath / iconFile
+            if iconPath.exists():
+                return iconPath
+        return self.programBaseDirPath / "Resources" / "Icons" / "Default.svg"
+
+    def addRecognisedEntityTypes(self, entityFile: Path) -> list:
+        entityTypesAdded = []
         try:
             tree = parse(entityFile, forbid_dtd=True, forbid_entities=True, forbid_external=True)
         except Exception as exc:
-            self.mainWindow.MESSAGEHANDLER.warning('Error occured when loading entities from '
-                                                   + entityFile + ': ' + str(exc) + ', skipping.')
-            return False
+            self.mainWindow.MESSAGEHANDLER.warning(
+                f'Error occurred when loading entities from {entityFile}: {exc}, skipping.')
+            return []
 
         root = tree.getroot()
 
         category = root.tag.replace('_', ' ')
         for entity in list(root):
-            entityName = entity.tag.replace('_', ' ')
-            attributes = entity.find('Attributes').text.strip().split(',')
-            icon = entity.find('Icon')
-            if icon is not None:
-                icon = icon.text.strip()
-            elif icon is None or icon == '':
-                icon = 'Default.svg'
-            if self.entityCategoryList.get(category) is None:
-                self.entityCategoryList[category] = {}
-            self.entityCategoryList[category][entityName] = {
-                'Attributes': attributes,
-                'Icon': str(Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / icon)}
-        return True
+            try:
+                entityName = entity.tag.replace('_', ' ')
+                attributes = entity.find('Attributes')
+                primaryCount = 0
+                attributesDict = {}
+                for attribute in list(attributes):
+                    defaultValue = attribute.attrib['default']
+                    valueCheck = attribute.attrib['check']
+                    isPrimary = attribute.attrib['primary'] == 'True'
+                    if isPrimary:
+                        if primaryCount > 0:
+                            raise AttributeError(f'Malformed Entity: {entityName} - too many primary fields')
+                        else:
+                            primaryCount += 1
+                    if not self.runCheckOnAttribute(defaultValue, valueCheck):
+                        raise AttributeError(f'Malformed Entity: {entityName} - default values do not pass their '
+                                             f'corresponding checks.')
+                    attributeName = attribute.text
+                    attributesDict[attributeName] = [attribute.attrib['default'], attribute.attrib['check'], isPrimary]
+                if primaryCount != 1:
+                    raise AttributeError(f'Malformed Entity: {entityName} - invalid number of primary fields '
+                                         f'specified.')
 
-    def loadCoreEntities(self):
-        entDir = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Core" / "Entities"
-        for entFile in listdir(entDir):
-            if entFile.endswith('.xml'):
-                self.addRecognisedEntityTypes(entDir / entFile)
+                icon = entity.find('Icon')
+                icon = icon.text.strip() if icon is not None else 'Default.svg'
+                if self.entityCategoryList.get(category) is None:
+                    self.entityCategoryList[category] = {}
+                self.entityCategoryList[category][entityName] = {
+                    'Attributes': attributesDict,
+                    'Icon': str(self.getIconPathForIconFile(icon))}
+                entityTypesAdded.append(f'{category}/{entityName}')
+            except (KeyError, AttributeError) as err:
+                # Ignore malformed entities
+                self.mainWindow.MESSAGEHANDLER.error(f'Error: {str(err)}', popUp=False)
+                continue
+        return entityTypesAdded
 
-    def loadModuleEntities(self):
-        entDir = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Modules"
-        for module in listdir(entDir):
-            for entFile in listdir(entDir / module):
+    def loadModuleEntities(self, modulePath: Path) -> list:
+        entitiesPath = modulePath / 'Entities'
+        allModuleEntitiesAdded = []
+        if entitiesPath.exists():
+            for entFile in listdir(entitiesPath):
                 if entFile.endswith('.xml'):
-                    self.addRecognisedEntityTypes(
-                        entDir / module / entFile)
+                    allModuleEntitiesAdded += self.addRecognisedEntityTypes(entitiesPath / entFile)
+        return allModuleEntitiesAdded
 
-    def getEntityJson(self, entityType: str, jsonData=None):
+    def loadModuleAssets(self, modulePath: Path):
+        moduleAssetsPath = modulePath / "Assets"
+        if moduleAssetsPath.exists():
+            self.moduleAssetPaths.append(moduleAssetsPath)
+            self.loadModuleBanners(modulePath)
+
+    def getEntityJson(self, entityType: str, jsonData=None) -> Union[dict, None]:
         eJson = {'uid': str(uuid4())}
+        if entityType in self.getAllEntitiesInCategory('Meta'):
+            eJson['uid'] += '@'
         try:
             for category in self.entityCategoryList:
                 if entityType in self.entityCategoryList[category]:
                     for attribute in self.entityCategoryList[category][entityType]['Attributes']:
-                        eJson[attribute] = str(None)
+                        eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
                     break
         except KeyError:
-            self.messageHandler.error("Attempted to get attributes for "
-                                      "malformed entity type: " + str(entityType), True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for malformed entity type: {entityType}", True)
             return None
         eJson['Entity Type'] = entityType
         eJson['Date Created'] = None
@@ -165,64 +323,84 @@ class ResourceHandler:
         eJson['Icon'] = self.getEntityDefaultPicture(entityType)
 
         if jsonData is not None:
-            for key in eJson:
+            # Allow setting of attributes that are not defined in the Entity specification.
+            for key in jsonData:
                 value = jsonData.get(key)
                 if value is not None and value != '':
                     eJson[key] = value
 
-        utcNow = datetime.isoformat(datetime.utcnow())
+        utcNow = datetime.isoformat(datetime.now(timezone.utc))
         if eJson['Date Created'] is None:
             eJson['Date Created'] = utcNow
+        else:
+            # Always make sure dates are in ISO format.
+            try:
+                eJson['Date Created'] = parser.parse(str(eJson['Date Created'])).isoformat()
+            except (TypeError, ValueError):
+                eJson['Date Created'] = utcNow
 
         eJson['Date Last Edited'] = utcNow
 
         return eJson
 
-    def getPrimaryFieldForEntityType(self, entityType: str):
+    def getPrimaryFieldForEntityType(self, entityType: str) -> Union[str, None]:
         try:
             for category in self.entityCategoryList:
                 if entityType in self.entityCategoryList[category]:
-                    return self.entityCategoryList[category][entityType]['Attributes'][0]
+                    for attribute in self.entityCategoryList[category][entityType]['Attributes']:
+                        if self.entityCategoryList[category][entityType]['Attributes'][attribute][2]:
+                            return attribute
         except KeyError:
-            self.messageHandler.error("Attempted to get primary attribute for "
-                                      "malformed entity type: " + str(entityType), True)
-            return None
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get primary attribute for malformed entity type: {entityType}", True)
+        return None
 
-    def getBareBonesEntityJson(self, entityType):
+    def getBareBonesEntityJson(self, entityType: str) -> Union[dict, None]:
         eJson = {}
         try:
             for category in self.entityCategoryList:
                 if entityType in self.entityCategoryList[category]:
                     for attribute in self.entityCategoryList[category][entityType]['Attributes']:
-                        eJson[attribute] = str(None)
+                        eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
                     break
         except KeyError:
-            self.messageHandler.error("Attempted to get attributes for "
-                                      "malformed entity type: " + str(entityType), True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for malformed entity type: {entityType}", True)
             return None
         eJson['Entity Type'] = entityType
 
         return eJson
 
-    def getLinkJson(self, jsonData):
+    def getLinkJson(self, jsonData: dict) -> Union[dict, None]:
         linkJson = {}
         try:
             linkJson['uid'] = jsonData['uid']
         except KeyError:
             return None
 
-        utcNow = datetime.isoformat(datetime.utcnow())
+        utcNow = datetime.isoformat(datetime.now(timezone.utc))
         linkJson['Resolution'] = str(jsonData.get('Resolution'))  # This way, if it is None, it is cast to a string.
         linkJson['Date Created'] = jsonData.get('Date Created')
+        # Make sure that dates are always in ISO format.
         if linkJson['Date Created'] is None:
             linkJson['Date Created'] = utcNow
+        else:
+            try:
+                linkJson['Date Created'] = parser.parse(str(linkJson['Date Created'])).isoformat()
+            except (TypeError, ValueError):
+                linkJson['Date Created'] = utcNow
         linkJson['Date Last Edited'] = utcNow
-        linkJson['Notes'] = str(jsonData.get('Notes'))
+        linkJson['Notes'] = str(jsonData.get('Notes', ""))
+
+        # Transfer all values from jsonData to linkJson, but preserve the values and order of linkJson for existing
+        #   keys.
+        jsonData |= linkJson
+        linkJson |= jsonData
 
         return linkJson
 
-    def getEntityDefaultPicture(self, entityType):
-        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Default.svg"
+    def getEntityDefaultPicture(self, entityType: str) -> QByteArray:
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Default.svg"
         try:
             for category in self.entityCategoryList:
                 if entityType in self.entityCategoryList[category]:
@@ -231,41 +409,215 @@ class ResourceHandler:
                         picture = entityPicture
                     break
         except KeyError:
-            self.messageHandler.warning("Attempted to get icon for "
-                                        "nonexistent entity type: " + str(entityType), popUp=False)
+            self.mainWindow.MESSAGEHANDLER.warning(
+                f"Attempted to get icon for nonexistent entity type: {entityType}", popUp=False)
         finally:
             with open(picture, 'rb') as pictureFile:
                 pictureContents = pictureFile.read()
-            pictureByteArray = QByteArray(pictureContents)
-            return pictureByteArray
+            return QByteArray(pictureContents)
 
     def getLinkPicture(self):
-        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Resolution.png"
-        return QIcon(str(picture)).pixmap(40, 40)
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Resolution.png"
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
 
     def getLinkArrowPicture(self):
-        picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Right-Arrow.svg"
-        return QIcon(str(picture)).pixmap(40, 40)
+        picture = self.programBaseDirPath / "Resources" / "Icons" / "Right-Arrow.svg"
+        return QtGui.QIcon(str(picture)).pixmap(40, 40)
 
     def deconstructGraph(self, graph: nx.DiGraph) -> tuple:
         nodes = {}
         for nodeKey in graph.nodes:
-            # Dereference the original dict so we don't actually convert its icon to data.
+            # Dereference the original dict, so we don't actually convert its icon to data.
             nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
-            try:
+            with contextlib.suppress(KeyError):
                 nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
-            except KeyError:
-                pass
-
         edges = {edgeKey: graph.edges.get(edgeKey) for edgeKey in graph.edges}
         return nodes, edges
 
-    def reconstructGraph(self, graphString: str) -> tuple:
+    def deconstructGraphForFileDump(self, graph: nx.DiGraph) -> tuple:
+        nodes = {}
+        for nodeKey in graph.nodes:
+            # Dereference the original dict, so we don't actually convert its icon to data.
+            nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
+            with contextlib.suppress(KeyError):
+                nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
+        edges = {str(edgeKey): graph.edges.get(edgeKey) for edgeKey in graph.edges}
+        return nodes, edges
+
+    def reconstructGraphFromString(self, graphString: str) -> tuple:
         nodes, edges = literal_eval(graphString)
         for node in nodes:
-            try:
+            with contextlib.suppress(KeyError):
                 nodes[node]['Icon'] = QByteArray(b64decode(nodes[node]['Icon']))
-            except KeyError:
-                pass
-
         return nodes, edges
+
+    def reconstructGraphFullFromFile(self, graphNodesAndEdges: Union[tuple, list]) -> nx.DiGraph:
+        returnGraph = nx.DiGraph()
+        graphNodes = graphNodesAndEdges[0]
+        graphEdges = graphNodesAndEdges[1]
+        for node in graphNodes:
+            with contextlib.suppress(KeyError):
+                graphNodes[node]['Icon'] = QByteArray(b64decode(graphNodes[node]['Icon']))
+            returnGraph.add_node(node, **graphNodes[node])
+
+        for edge in graphEdges:
+            edgeUID = tuple(literal_eval(edge))
+            graphEdges[edge]['uid'] = edgeUID
+            returnGraph.add_edge(*edgeUID, **graphEdges[edge])
+
+        return returnGraph
+
+
+class StringPropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(StringPropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+
+    def getValue(self):
+        return self.text()
+
+
+class FilePropertyInput(QtWidgets.QLineEdit):
+
+    def __init__(self, placeholderText, defaultText):
+        super(FilePropertyInput, self).__init__()
+        self.setPlaceholderText(placeholderText)
+        if defaultText is not None:
+            self.setText(defaultText)
+        self.fileDialog = QtWidgets.QFileDialog()
+
+    def getValue(self):
+        return self.text()
+
+    def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
+        fileChosen = self.fileDialog.getOpenFileName(self,
+                                                     "Open File",
+                                                     str(Path.home()),
+                                                     options=QtWidgets.QFileDialog.Option.DontUseNativeDialog)
+        self.setText(fileChosen[0])
+
+
+class SingleChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOption):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(SingleChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOption is None:
+            defaultOption = ''
+
+        for option in enforceOptionsSet:
+            radioButton = QtWidgets.QRadioButton(option)
+            if option == defaultOption:
+                radioButton.setChecked(True)
+            else:
+                radioButton.setChecked(False)
+            self.options.append(radioButton)
+            vboxLayout.addWidget(radioButton)
+
+    def getValue(self):
+        return next((option.text() for option in self.options if option.isChecked()), '')
+
+
+class MultiChoicePropertyInput(QtWidgets.QGroupBox):
+
+    def __init__(self, optionsSet: set, defaultOptions):
+        # Ensure that the options given are an actual set (i.e. each one is unique)
+        enforceOptionsSet = set(optionsSet)
+        super(MultiChoicePropertyInput, self).__init__(title='Option Selection')
+        vboxLayout = QtWidgets.QVBoxLayout()
+        self.setLayout(vboxLayout)
+
+        self.options = []
+        if defaultOptions is None:
+            defaultOptions = []
+
+        for option in enforceOptionsSet:
+            checkBox = QtWidgets.QCheckBox(option)
+            if option in defaultOptions:
+                checkBox.setChecked(True)
+            else:
+                checkBox.setChecked(False)
+            self.options.append(checkBox)
+            vboxLayout.addWidget(checkBox)
+
+    def getValue(self):
+        return [option.text() for option in self.options if option.isChecked()]
+
+
+class MinSizeStackedLayout(QtWidgets.QStackedLayout):
+    """
+    Resize the layout to always take up the appropriate space for the currently selected widget.
+    Otherwise, large widgets (due to selecting entities with long strings of text) will stretch
+    out the ScrollArea and make the other, non-selected widgets to look bad when the layout
+    switches over.
+
+    https://stackoverflow.com/a/34300567
+    """
+
+    def sizeHint(self) -> QSize:
+        return self.currentWidget().sizeHint()
+
+    def minimumSize(self) -> QSize:
+        return self.currentWidget().minimumSize()
+
+
+class RichNotesEditor(QtWidgets.QTextBrowser):
+
+    def __init__(self, parent=None, currentText: str = '#### Type notes here.\n', allowEditing: bool = True):
+        super(RichNotesEditor, self).__init__(parent=parent)
+        self.allowEditing = allowEditing
+        self.setReadOnly(True)
+        if allowEditing:
+            self.setUndoRedoEnabled(True)
+            self.setTextInteractionFlags(Qt.TextBrowserInteraction | Qt.TextSelectableByKeyboard)
+        else:
+            self.setUndoRedoEnabled(False)
+            self.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.TextSelectableByKeyboard
+                                         | Qt.LinksAccessibleByMouse | Qt.LinksAccessibleByKeyboard)
+
+        self.contents = currentText
+        self.setMarkdown(self.contents)
+        self.textFormat = self.currentCharFormat()
+
+    def startEditing(self) -> None:
+        if self.allowEditing and self.isReadOnly():
+            # Reset char format to plain text.
+            self.setCurrentCharFormat(self.textFormat)
+            self.setPlainText(self.contents)
+            self.setReadOnly(False)
+
+    def stopEditing(self) -> None:
+        if not self.isReadOnly():
+            self.contents = self.toPlainText()
+            self.setMarkdown(self.contents)
+            self.setReadOnly(True)
+
+    def dropEvent(self, e: QtGui.QDropEvent) -> None:
+        if self.allowEditing:
+            editingBefore = self.isReadOnly()
+            self.startEditing()
+            super(RichNotesEditor, self).dropEvent(e)
+            if editingBefore:
+                self.stopEditing()
+
+    def mousePressEvent(self, ev: QtGui.QMouseEvent) -> None:
+        potentialLink = self.anchorAt(ev.pos())
+        if not potentialLink and ev.button() == QtGui.Qt.MouseButton.LeftButton:
+            self.startEditing()
+        super(RichNotesEditor, self).mousePressEvent(ev)
+
+    def focusOutEvent(self, ev: QtGui.QFocusEvent) -> None:
+        if not self.underMouse() and self.isActiveWindow():
+            self.stopEditing()
+        super(RichNotesEditor, self).focusOutEvent(ev)
+
+    def doSetSource(self, name: Union[QUrl, str], resourceType: QtGui.QTextDocument.ResourceType = ...) -> None:
+        QtGui.QDesktopServices.openUrl(name)

Core/SettingsObject.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
-from pickle import dump
+from msgpack import dump
 from shutil import move
 from pathlib import Path
 from Core.PathHelper import is_path_exists_or_creatable_portable
+from PySide6.QtCore import QSettings
 
 
 class SettingsObject(dict):
@@ -21,12 +22,67 @@ class SettingsObject(dict):
 
     def __init__(self):
         super().__init__()
+        self.globalSettings = QSettings()
+        self.globalSettings.setValue("Program/Version", "v1.6.5")
+        self.globalSettings.setValue("Program/TOR Profile Location",
+                                     self.globalSettings.value("Program/TOR Profile Location", ""))
+        self.globalSettings.setValue("Program/BaseDir",
+                                     self.globalSettings.value("Program/BaseDir", "Unset"))
+        self.globalSettings.setValue("Program/Version Check Source",
+                                     self.globalSettings.value("Program/Version Check Source",
+                                                               "https://api.github.com/repos/AccentuSoft/LinkScope_Client/releases/latest"))
+        self.globalSettings.setValue("Program/Update Source",
+                                     self.globalSettings.value("Program/Update Source",
+                                                               "https://github.com/AccentuSoft/LinkScope_Client/releases/latest/download/"))
+
+        # The value '20' equates to logging.INFO
+        # It's not necessary to set this, but we will for
+        #   the sake of completeness
+        self.globalSettings.setValue("Logging/Severity",
+                                     self.globalSettings.value("Logging/Severity", "20"))
+        self.globalSettings.setValue("Logging/Logfile",
+                                     self.globalSettings.value("Logging/Logfile",
+                                                               str(Path.home() / 'LinkScope_logfile.log')))
+
+        self.globalSettings.setValue("Program/Graph Layout",
+                                     self.globalSettings.value("Program/Graph Layout", "dot"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Type",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Type", "Mono"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Size",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Size", "11"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Font Boldness",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Font Boldness", "700"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Type",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Type", "Mono"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Size",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Size", "11"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Font Boldness",
+                                     self.globalSettings.value("Program/Graphics/Link Text Font Boldness", "700"))
+        self.globalSettings.setValue("Program/Graphics/Entity Text Color",
+                                     self.globalSettings.value("Program/Graphics/Entity Text Color", "#000000"))
+        self.globalSettings.setValue("Program/Graphics/Link Text Color",
+                                     self.globalSettings.value("Program/Graphics/Link Text Color", "#000000"))
+        self.globalSettings.setValue("Program/Graphics/Label Fade Scroll Distance",
+                                     self.globalSettings.value("Program/Graphics/Label Fade Scroll Distance", "3"))
+
+        self.globalSettings.setValue("Program/Usage/First Time Start",
+                                     self.globalSettings.value("Program/Usage/First Time Start", 'true'))
+
+        self.globalSettings.setValue("Program/Sources/Sources List",
+                                     self.globalSettings.value(
+                                         "Program/Sources/Sources List",
+                                         {}))
+
+        self.globalSettings.setValue("Program/Sources/Module Packs List",
+                                     self.globalSettings.value(
+                                         "Program/Sources/Module Packs List",
+                                         {}))
+
         self.setValue("Project/Name", "Untitled")
-        self.setValue("Program/BaseDir", "Unset")  # dirname(abspath(getsourcefile(lambda:0))) + "/../" )
         self.setValue("Project/BaseDir", "")
         self.setValue("Project/FilesDir", "")
         # For any entity with a Path variable, this dictates whether a copy of the original is made or whether a
-        #   symlink is created. Symlinks require special permissions or developer mode in Windows however.
+        #   symlink is created. Symlinks however require special permissions or developer mode in Windows.
         # To ensure that the software works out-of-the-box on all platforms, the default is set to 'Copy'.
         self.setValue("Project/Symlink or Copy Materials", "Copy")  # Values are 'Copy' or 'Symlink'.
         self.setValue("Project/Resolution Result Grouping Threshold", "15")
@@ -34,26 +90,61 @@ class SettingsObject(dict):
         self.setValue("Project/Question Answering Retriever Value", "10")
         self.setValue("Project/Question Answering Reader Value", "10")
         self.setValue("Project/Server/Project", "")
+        self.setValue("Project/Server/Collectors", "{}")
 
-        # The value '20' equates to logging.INFO
-        # It's not necessary to set this, but we will for
-        #   the sake of completeness
-        self.setValue("Logging/Severity", "20")
-        self.setValue("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log'))
+    def getGroupSettings(self, settingsGroup: str) -> dict:
+        if not settingsGroup.endswith('/'):
+            settingsGroup += '/'
+        settingsDict = {
+            setting: self.globalSettings.value(setting)
+            for setting in self.globalSettings.allKeys()
+            if setting.startswith(settingsGroup)
+        }
+        for setting in self:
+            if setting.startswith(settingsGroup):
+                settingsDict[setting] = self[setting]
+        return dict(sorted(settingsDict.items()))
 
-    # Usability Alias
-    def setValue(self, key, value):
+    def setValue(self, key, value) -> None:
+        if self.globalSettings.contains(key):
+            self.globalSettings.setValue(key, value)
         self[key] = value
 
+    def setGlobalValue(self, key, value) -> None:
+        """
+        Helper in the case we want to be explicit in setting a value globally.
+        """
+        self.globalSettings.setValue(key, value)
+
     def value(self, key, alt=None):
+        if self.globalSettings.contains(key):
+            return self.globalSettings.value(key)
         return self.get(key, alt)
 
-    def save(self):
+    def removeKey(self, key) -> bool:
+        try:
+            if self.globalSettings.contains(key):
+                self.globalSettings.remove(key)
+            else:
+                self.pop(key)
+            return True
+        except KeyError:
+            return False
+
+    def save(self) -> None:
         # Save and then move to prevent corruption if the application closes unexpectedly.
         actualSavePath = str(Path(self.value("Project/BaseDir")).joinpath(self.value("Project/Name") + ".linkscope"))
         if is_path_exists_or_creatable_portable(actualSavePath):
-            tempSavePath = actualSavePath + '.tmp'
-            projectFile = open(tempSavePath, "wb")
-            dump(self, projectFile)
-            projectFile.close()
+            tempSavePath = f'{actualSavePath}.tmp'
+            with open(tempSavePath, "wb") as projectFile:
+                dump(self, projectFile)
             move(tempSavePath, actualSavePath)
+        self.globalSettings.sync()
+        globalSettingsSavingError = self.globalSettings.status()
+        if globalSettingsSavingError != self.globalSettings.Status.NoError:
+            raise ValueError(f'Could not save global settings: {globalSettingsSavingError}')
+
+    def load(self, savedDict: dict) -> None:
+        # No need to do anything with global settings.
+        for key in savedDict:
+            self[key] = savedDict[key]

Core/URLManager.py

@@ -5,8 +5,8 @@ import magic
 from pathlib import Path
 from os import symlink
 from shutil import copy2
-from hashlib import md5
-from binascii import hexlify
+from hashlib import sha3_512
+from urllib.parse import urlparse
 
 from PySide6 import QtCore
 
@@ -30,19 +30,11 @@ class URLManager:
         Takes a list of QUrls and returns a list of entities that correspond
         to them.
         """
-        if len(urls) == 1:
-            url = urls[0]
-            entityJson = self.handleURL(url)
-            return [entityJson]
-
-        returnValue = []
-        for url in urls:
-            returnValue += [self.handleURL(url)]
-
-        return returnValue
+        return [self.handleURL(url) for url in urls]
 
     def handleURL(self, url):
-        if not url.isValid():
+        parsedURL = urlparse(url.toString())
+        if not url.isValid() or (not all([parsedURL.scheme, parsedURL.netloc]) and parsedURL.scheme != 'file'):
             return None
         if url.isLocalFile():
             return self.handleLocalURL(url)
@@ -50,7 +42,7 @@ class URLManager:
             return self.handleRemoteURL(url)
 
     def handleURLString(self, urlString):
-        self.handleURL(QtCore.QUrl(urlString))
+        return self.handleURL(QtCore.QUrl(urlString))
 
     def handleLocalURL(self, url):
         urlPath = Path(url.toLocalFile())
@@ -62,20 +54,25 @@ class URLManager:
         if savePathString == 'None':
             return None
 
+        fileType = magic.from_file(urlPathString, mime=True)
+        fileTypeSplit1, fileTypeSplit2 = fileType.split('/', 1)
+        # CSV files not considered - may have any dialect, hard to accommodate.
+        if urlPath.suffix in {'.ods', '.xls', '.xlsm', '.xlsx'} and \
+                fileTypeSplit2 in {'vnd.oasis.opendocument.spreadsheet',
+                                   'vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+                                   'vnd.ms-excel'}:
+            return {"Spreadsheet Name": urlName,
+                    "File Path": savePathString,
+                    "Entity Type": "Spreadsheet"}
         # Only support zip files for archives (for now) 10/Jul/2021).
-        if zipfile.is_zipfile(urlPathString):
-            entityJson = {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
+        elif zipfile.is_zipfile(urlPathString):
+            return {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
+        elif fileTypeSplit1 == "video":
+            return {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
+        elif fileTypeSplit1 == "image":
+            return {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
         else:
-            fileType = magic.from_file(urlPathString, mime=True).split('/')[0]
-            if fileType == "video":
-                entityJson = {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
-                pass
-            elif fileType == "image":
-                entityJson = {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
-                pass
-            else:
-                entityJson = {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
-        return entityJson
+            return {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
 
     def moveURLToProjectFilesHelperIfNeeded(self, urlPath: Path):
         valuePath = Path(urlPath).absolute()
@@ -88,12 +85,12 @@ class URLManager:
             savePath = valuePath.relative_to(projectFilesPath)
         except ValueError:
             # The file selected is not in Project Files
-            createSymlink = True if self.mainWindow.SETTINGS.value("Project/Symlink or Copy Materials") == "Symlink" \
-                else False
+            createSymlink = self.mainWindow.SETTINGS.value("Project/Symlink or Copy Materials") == "Symlink"
+
             projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir"))
             # Create a unique path in Project Files
-            saveHash = hexlify(md5(str(urlPath).encode()).digest()).decode()[:8]
-            savePath = projectFilesPath / (saveHash + '|' + urlPath.name)
+            saveHash = sha3_512(str(urlPath).encode()).hexdigest()[:16]  # nosec
+            savePath = projectFilesPath / f'{saveHash}_{urlPath.name}'
 
             if createSymlink:
                 symlink(urlPath, savePath)
@@ -105,5 +102,7 @@ class URLManager:
         return savePath
 
     def handleRemoteURL(self, url):
-        entity = {'Entity Type': 'Website', 'URL': url.toString()}
-        return entity
+        stringURL = url.toString()
+        return {'Entity Type': 'Onion Website', 'Onion URL': stringURL} \
+            if self.mainWindow.RESOURCEHANDLER.runCheckOnAttribute(stringURL, 'Onion') \
+            else {'Entity Type': 'Website', 'URL': stringURL}

Core/UpdateManager.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+
+import contextlib
+import platform
+import subprocess
+import requests
+import tempfile
+import shutil
+import os
+import ctypes
+
+from pathlib import Path
+from semver import compare
+from PySide6 import QtCore, QtWidgets
+
+
+class UpdateManager:
+
+    def __init__(self, mainWindow):
+        self.updateThread = None
+        self.mainWindow = mainWindow
+        self.system = platform.system()
+        if self.system == 'Windows':
+            self.baseSoftwarePath = Path(os.path.abspath(os.sep)) / 'Program Files' / 'LinkScope'
+        else:
+            self.baseSoftwarePath = Path(os.path.abspath(os.sep)) / 'usr' / 'local' / 'sbin' / 'LinkScope'
+
+    def getDownloadURL(self):
+        downloadURLBase = self.mainWindow.SETTINGS.value("Program/Update Source")
+        if self.system == 'Windows':
+            return f"{downloadURLBase}LinkScope-Windows-x64.7z"
+        else:
+            return f"{downloadURLBase}LinkScope-Ubuntu-x64.7z"
+
+    def getLatestVersion(self):
+        with contextlib.suppress(Exception):
+            version_req = requests.get(self.mainWindow.SETTINGS.value("Program/Version Check Source"),
+                                       headers={'User-Agent': 'LinkScope Update Checker'})
+            if version_req.status_code == 200:
+                return version_req.json()['tag_name']
+        return None
+
+    def isUpdateAvailable(self):
+        latest_version = self.getLatestVersion()
+        return (
+                latest_version is not None
+                and compare(self.mainWindow.SETTINGS.value("Program/Version").lstrip('v'),
+                            latest_version.lstrip('v')) < 0
+        )
+
+    def doUpdate(self) -> None:
+        if self.updateThread is not None:
+            self.mainWindow.MESSAGEHANDLER.error('Update already in progress.')
+            return
+        self.mainWindow.MESSAGEHANDLER.info('Starting update...')
+        self.updateThread = UpdaterThread(self, self.mainWindow)
+        self.updateThread.updateDoneSignal.connect(self.finalizeUpdate)
+        self.updateThread.start()
+        self.mainWindow.MESSAGEHANDLER.info('Update in progress')
+
+    def finalizeUpdate(self, updateTempPath: str) -> None:
+        if updateTempPath != '':
+            latest_version = self.getLatestVersion()
+            self.mainWindow.SETTINGS.setValue("Program/Version", latest_version)
+            self.mainWindow.MESSAGEHANDLER.info('Updating done, please restart for the changes to take effect.')
+            self.mainWindow.MESSAGEHANDLER.info("The application will now save and close to apply the updates. "
+                                                "Please wait for a few minutes before reopening LinkScope.",
+                                                popUp=True)
+
+            uncompressNewVersion(
+                self.system,
+                self.mainWindow.SETTINGS.value("Program/BaseDir"),
+                str(self.baseSoftwarePath),
+                updateTempPath)
+            self.mainWindow.close()
+
+        else:
+            self.mainWindow.MESSAGEHANDLER.info('Updating failed.')
+
+
+class UpdaterWindow(QtWidgets.QDialog):
+
+    def __init__(self, mainWindow, updateManager: UpdateManager, updateAvailableOverride: bool = False):
+        super().__init__()
+        self.mainWindow = mainWindow
+        self.updateManager = updateManager
+
+        self.setWindowTitle('Update Manager')
+
+        layout = QtWidgets.QGridLayout()
+        self.setLayout(layout)
+
+        doUpdateButton = QtWidgets.QPushButton('Update')
+        doUpdateButton.clicked.connect(self.initiateUpdate)
+        cancelButton = QtWidgets.QPushButton('Close')
+        cancelButton.clicked.connect(self.reject)
+
+        updateAvailableLabel = QtWidgets.QLabel()
+        updateAvailableLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        if updateAvailableOverride or updateManager.isUpdateAvailable():
+            updateAvailableLabel.setText('An update for LinkScope is available.')
+        else:
+            updateAvailableLabel.setText('LinkScope is up to date.')
+            doUpdateButton.setDisabled(True)
+            doUpdateButton.setEnabled(False)
+
+        layout.addWidget(updateAvailableLabel, 1, 1, 1, 2)
+        layout.addWidget(cancelButton, 2, 1)
+        layout.addWidget(doUpdateButton, 2, 2)
+
+    def initiateUpdate(self) -> None:
+        self.updateManager.doUpdate()
+        self.accept()
+
+
+class UpdaterThread(QtCore.QThread):
+    updateDoneSignal = QtCore.Signal(str)
+
+    def __init__(self, updateManager, mainWindow):
+        super().__init__()
+        self.mainWindow = mainWindow
+        self.updateManager = updateManager
+
+    def run(self) -> None:
+        try:
+            downloadUrl = self.updateManager.getDownloadURL()
+            clientTempCompressedArchive = tempfile.mkstemp(suffix='.7z')
+            tempPath = clientTempCompressedArchive[1]
+
+            with os.fdopen(clientTempCompressedArchive[0], 'wb') as tempArchive:
+                with requests.get(downloadUrl, stream=True) as fileStream:
+                    for chunk in fileStream.iter_content(chunk_size=5 * 1024 * 1024):
+                        tempArchive.write(chunk)
+            self.updateDoneSignal.emit(tempPath)
+        except Exception:
+            self.updateDoneSignal.emit('')
+
+
+def uncompressNewVersion(system: str, baseDir: str, baseSoftwarePath: str, updateTempPath: str):
+    tempDir = tempfile.mkdtemp(prefix='LinkScope_Updater_TMP_')
+
+    if system == 'Windows':
+        updaterPath = Path(baseDir) / "UpdaterUtil.exe"
+
+        tempUpdaterPath = Path(tempDir) / updaterPath.name
+        shutil.copy(updaterPath, tempUpdaterPath)
+
+        # This is done so that Windows spawns the updater as a detached process.
+        ShellExecuteEx = ctypes.windll.shell32.ShellExecuteEx
+        SEE_MASK_NO_CONSOLE = 0x00008000
+
+        class SHELLEXECUTEINFO(ctypes.Structure):
+            _fields_ = [
+                ("cbSize", ctypes.c_ulong),
+                ("fMask", ctypes.c_ulong),
+                ("hwnd", ctypes.c_void_p),
+                ("lpVerb", ctypes.c_char_p),
+                ("lpFile", ctypes.c_char_p),
+                ("lpParameters", ctypes.c_char_p),
+                ("lpDirectory", ctypes.c_char_p),
+                ("nShow", ctypes.c_int),
+                ("hInstApp", ctypes.c_void_p),
+                ("lpIDList", ctypes.c_void_p),
+                ("lpClass", ctypes.c_char_p),
+                ("hkeyClass", ctypes.c_void_p),
+                ("dwHotKey", ctypes.c_ulong),
+                ("hIconOrMonitor", ctypes.c_void_p),
+                ("hProcess", ctypes.c_void_p),
+            ]
+
+        sei = SHELLEXECUTEINFO()
+        sei.cbSize = ctypes.sizeof(sei)
+        sei.fMask = SEE_MASK_NO_CONSOLE
+        sei.lpVerb = b"runas"
+        sei.lpFile = bytes(tempUpdaterPath)
+        sei.lpParameters = f'"{updateTempPath}" "{baseSoftwarePath}"'.encode('utf-8')
+        sei.nShow = 1
+
+        if not ShellExecuteEx(ctypes.byref(sei)):
+            raise ctypes.WinError()
+
+    elif system == 'Linux':
+        updaterPath = Path(baseDir) / "UpdaterUtil"
+
+        tempUpdaterPath = Path(tempDir) / updaterPath.name
+        shutil.copy(updaterPath, tempUpdaterPath)
+
+        subprocess.Popen(
+            f'pkexec "{tempUpdaterPath}" "{updateTempPath}" "{baseSoftwarePath}"',
+            start_new_session=True,
+            close_fds=True,
+            shell=True,
+        )

Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByDealer:
-    name = "Aircraft Inquiry By Dealer"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Company"}
-    resultTypes = {'Phrase', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import pandas as pd
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}DealerResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Dealertxt": entity['Company Name']}))
-            for future in as_completed(futures):
-                uid = uidList[futures.index(future)]
-                try:
-                    df_list = pd.read_html(future.result().text)
-                except requests.exceptions.ConnectionError:
-                    return "Please check your internet connection"
-                except ValueError:
-                    return_result = []
-                    return return_result
-                df = df_list[0]
-                for certificate_index in range(len(df["Certificate Number"])):
-                    index_of_child = len(return_result)
-                    return_result.append([{'Company Name': df["Name"][certificate_index],
-                                           'Entity Type': 'Company'},
-                                          {uid: {'Resolution': 'Aircraft Dealer', 'Notes': ''}}])
-                    return_result.append([{'Phrase': df["Certificate Number"][certificate_index],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Aircraft Certificate Number', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByEngine:
-    name = "Aircraft Inquiry By Engine"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Phrase"}
-    resultTypes = {"Phrase"}
-    parameters = {'Manufacturer': {'description': "Enter the Manufacturer of the Engine Model",
-                                   'type': 'String',
-                                   'value': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        Manufacturer = parameters['Manufacturer']
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}EngineReferenceResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Modeltxt": entity['Phrase'],
-                                                               "MfrNametxt": Manufacturer}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            return_result.append([{'Phrase': f"Model Code:str({df['Mfr/Mdl Code']})",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Model Code', 'Notes': ''}}])
-            return_result.append([{'Phrase': f"Engine Type:{df['Type Engine']}",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Engine Type', 'Notes': ''}}])
-            return_result.append([{'Phrase': f"Horse Power:str({df['Horsepower']})",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Engine Horsepower', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByNNumber:
-    name = "Aircraft Inquiry By N-Number"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Phrase"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}NNumberResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"NNumbertxt": entity['Phrase']}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df1 = df_list[0]
-            df2 = df_list[1]
-            df3 = df_list[2]
-            return_result.append([{'ID Number': df1[1][0],
-                                   'Entity Type': 'Identification Number'},
-                                  {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
-            return_result.append([{'Company Name': df1[1][1],
-                                   'Entity Type': 'Company'},
-                                  {uid: {'Resolution': 'Aircraft Company', 'Notes': ''}}])
-            return_result.append([{'Full Name': df2[1][0],
-                                   'Entity Type': 'Person'},
-                                  {uid: {'Resolution': 'Aircraft Owner', 'Notes': ''}}])
-            return_result.append([{'City Name': df2[1][2],
-                                   'Entity Type': 'City'},
-                                  {uid: {'Resolution': "Aircraft Owner's City", 'Notes': ''}}])
-            return_result.append([{'Country Name': df2[1][4],
-                                   'Entity Type': 'Country'},
-                                  {uid: {'Resolution': "Aircraft Owner's Country", 'Notes': ''}}])
-            return_result.append([{'Phrase': df3[1][1],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': "Aircraft Engine Series", 'Notes': ''}}])
-            return_result.append([{'Phrase': df3[1][2],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': "Aircraft Engine Motor", 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByPersonName:
-    name = "Aircraft Inquiry By Person Name"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Person"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}NameResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"nametxt": entity['Full Name'], "sort_option": "1"}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            for i in range(len(df["N-Number"])):
-                return_result.append([{'Phrase': df["N-Number"][0],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
-                return_result.append([{'ID Number': str(df['Serial Number'][0]),
-                                       'Entity Type': 'Identification Number'},
-                                      {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
-                return_result.append([{'Company Name': df['Manufacturer Name Model'][0],
-                                       'Entity Type': 'Company'},
-                                      {uid: {'Resolution': 'Aircraft Manufacturer Name', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryBySerialNumber:
-    name = "AircraftInquiry By Serial Number"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Identification Number"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}SerialResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Serialtxt": entity['ID Number'], "sort_option": "1"}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            for i in range(len(df["N-Number"])):
-                return_result.append([{'Company Name': df["Manufacturer Name"][i],
-                                       'Entity Type': 'Company'},
-                                      {uid: {'Resolution': 'Aircraft Manufacturer', 'Notes': ''}}])
-                return_result.append([{'Phrase': df["N-Number"][i],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
-                return_result.append([{'Phrase': df["Model"][i],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft Model', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/requirements.txt

@@ -1,2 +0,0 @@
-requests
-pandas

Modules/Aleph/Aleph.xml

@@ -1,12 +0,0 @@
-<Aleph>
-  <Aleph_ID>
-    <Attributes>
-      ID
-    </Attributes>
-  </Aleph_ID>
-  <Aleph_Collection_ID>
-    <Attributes>
-      ID
-    </Attributes>
-  </Aleph_Collection_ID>
-</Aleph>

Modules/Aleph/Aleph_Entity_Search.py

@@ -1,269 +0,0 @@
-#!/usr/bin/env python3
-
-
-class Aleph_Entity_Search:
-    name = "Aleph Entity Search"
-    description = "Find information about a given search parameter"
-    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
-    resultTypes = {'Phrase'}
-    parameters = {'Number of results': {'description': 'Creating a lot of nodes could slow down the software. Please '
-                                                       'be mindful of the value you enter.',
-                                        'type': 'String',
-                                        'value': 'Enter the number of results you want returned',
-                                        'default': '1'},
-                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}
-                  }
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import pycountry
-        import time
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        return_result = []
-        uidList = []
-        futures = []
-
-        url = "https://aleph.occrp.org/api/2/entities"
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        gender = "None"
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        try:
-            max_results = int(parameters['Number of results'])
-        except ValueError:
-            return "The value for parameter 'Max Results' is not a valid integer."
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                crafted_url = url + f"?q={primary_field}&filter:schemata=Thing&limit={max_results}"
-                time.sleep(1)
-                futures.append(session.get(crafted_url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            print(response)
-            for schema in response['results']:
-                index_of_child = len(return_result)
-                try:
-                    if schema['schema'] == "Person":
-                        if schema['properties'].get('gender') is not None \
-                                and schema['properties'].get('gender')[0] == "F":
-                            gender = "Female"
-                        elif schema['properties'].get('gender') is not None \
-                                and schema['properties'].get('gender')[0] == "M":
-                            gender = "Male"
-                        if schema['properties'].get('legalForm') is not None:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties'].get('birthDate')),
-                                  'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}",
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Full Name': str(schema['properties']['name'][0]),
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['birthDate'][0]),
-                                  'Notes': schema['links']['self'],
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
-                        if schema['properties'].get('registrationNumber') is not None:
-                            return_result.append(
-                                [{'Registration Number': str(schema['properties']['registrationNumber'][0]),
-                                  'Notes': '',
-                                  'Entity Type': 'Company'},
-                                 {index_of_child: {'Resolution': 'Aleph Registration Number', 'Notes': ''}}])
-                        if schema['properties'].get('country') is not None:
-                            return_result.append(
-                                [{'Country Name': str(
-                                    pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
-                                    'Notes': '',
-                                    'Entity Type': 'Country'},
-                                    {index_of_child: {'Resolution': 'Country of Origin', 'Notes': ''}}])
-                        if schema['properties'].get('addressEntity'):
-                            return_result.append(
-                                [{'Street Address': str(
-                                    schema['properties']['addressEntity'][0]['properties']['full'][0]),
-                                    'Notes': '',
-                                    'Entity Type': 'Address'},
-                                    {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Street Address': str(schema['properties']['address'][0]),
-                                  'Notes': '',
-                                  'Entity Type': 'Address'},
-                                 {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph Collection ID'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                    elif schema['schema'] == "Organization":
-                        return_result.append(
-                            [{'Organization Name': str(schema['properties']['name'][0]),
-                              'Registration Number': str(schema['properties']['registrationNumber'][0]),
-                              'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}\n"
-                                       f"Source URL: {schema['properties']['sourceUrl'][0]}",
-                              'Entity Type': 'Organization'},
-                             {uid: {'Resolution': 'Aleph Organisation Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Country Name': str(
-                                pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
-                                'Notes': '',
-                                'Entity Type': 'Country'},
-                                {index_of_child: {'Resolution': "Aleph Organisation Country", 'Notes': ''}}])
-                        return_result.append(
-                            [{'Street Address': str(schema['properties']['address'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Address'},
-                             {index_of_child: {'Resolution': "Aleph Organisation Address", 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': "Aleph Organisation ID", 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phone Number': str(schema['properties']['phone'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Phone Number'},
-                             {index_of_child: {'Resolution': 'Phone Number', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['classification'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Organisation Classification', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                    elif schema['schema'] == "Pages":
-                        if 'updated_at' in schema:
-                            date_created = schema['updated_at']
-                        else:
-                            date_created = schema['created_at']
-                        doc_name = 'Document: ' + schema['properties']['title'][0]
-                        entity_link = schema['links']['self']
-                        file_link = schema['links']['file']
-                        source_url = schema['properties']['sourceUrl'][0]
-                        return_result.append(
-                            [{'Phrase': doc_name,
-                              'Source': source_url,
-                              'Notes': 'Link to Aleph Entity: ' + entity_link + '\n\n' +
-                                       'Link to document: ' + file_link,
-                              'Entity Type': 'Phrase',
-                              'Date Created': date_created},
-                             {uid: {'Resolution': 'Aleph Document', 'Notes': ''}}])
-                    elif schema['properties']['parent'][0]['schema'] == "Person":
-                        gender = str(schema['properties']['parent'][0]['properties'].get('gender')[0])
-                        if schema['properties']['parent'][0]['properties'].get('legalForm') is not None:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
-                                  'Notes': f"{schema['properties']['parent'][0]['links']['self']}\nLegal Form: "
-                                           f"{schema['properties']['parent'][0]['properties']['legalForm'][0]}",
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
-                                  'Notes': schema['properties']['parent'][0]['links']['self'],
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
-                        if schema['properties']['parent'][0]['properties'].get('registrationNumber') is not None:
-                            return_result.append(
-                                [{'Registration Number': str(
-                                    schema['properties']['parent'][0]['properties']['registrationNumber'][0]),
-                                    'Notes': '',
-                                    'Entity Type': 'Company'},
-                                    {index_of_child: {'Resolution': 'Company Registration Number', 'Notes': ''}}])
-                        if schema['properties']['parent'][0]['properties'].get('country') is not None:
-                            return_result.append(
-                                [{'Country Name': str(
-                                    pycountry.countries.get(
-                                        alpha_2=schema['properties']['parent'][0]['properties']['country'][0]).name),
-                                    'Notes': '',
-                                    'Entity Type': 'Country'},
-                                    {index_of_child: {'Resolution': 'Country', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['properties']['parent'][0]['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['label']),
-                              'Notes': str(schema['properties']['parent'][0]['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                        index_of_child_of_child = len(return_result)
-                        return_result.append(
-                            [{'Company Name': str(schema['properties']['name'][0]),
-                              'Notes': schema['links']['self'],
-                              'Entity Type': 'Company'},
-                             {index_of_child: {'Resolution': 'Aleph Company Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Street Address': str(schema['properties']['addressEntity'][0]['properties']['full'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Address'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Person Address', 'Notes': ''}}])
-                        for country_code in schema['collection']['countries']:
-                            return_result.append(
-                                [{'Country Name': str(pycountry.countries.get(alpha_2=country_code).name),
-                                  'Notes': '',
-                                  'Entity Type': 'Country'},
-                                 {index_of_child_of_child: {'Resolution': 'Country', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph Collection ID'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Entity Search', 'Notes': ''}}])
-                except (TypeError, KeyError) as e:
-                    print(repr(e))
-                    continue
-        return return_result

Modules/Aleph/GetCollectionByID.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetCollectionByID:
-    name = "Get Collections By ID"
-    description = "Find information about Collections and their IDs"
-    originTypes = {'Phrase'}
-    resultTypes = {'Phrase. Person, Address, Phone Number, Email Address, Country, Bank Account'}
-    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                url = f"https://aleph.occrp.org/api/2/collections/{primary_field}"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            if response['statistics']['names'].get('values') is not None:
-                nameKeys = list(response['statistics']['names'].get('values').keys())
-                for nameKey in nameKeys:
-                    returnResults.append([{'Full Name': str(nameKey),
-                                           'Entity Type': 'Person'},
-                                          {uid: {'Resolution': 'Person Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['addresses'].get('values') is not None:
-                addressKeys = list(response['statistics']['addresses'].get('values').keys())
-                for addressKey in addressKeys:
-                    returnResults.append([{'Street Address': str(addressKey),
-                                           'Entity Type': 'Address'},
-                                          {uid: {'Resolution': 'Address Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['phones'].get('values') is not None:
-                phoneKeys = list(response['statistics']['phones'].get('values').keys())
-                for phoneKey in phoneKeys:
-                    returnResults.append([{'Phone Number': str(phoneKey),
-                                           'Entity Type': 'Phone Number'},
-                                          {uid: {'Resolution': 'Phone Number Entity',
-                                                 'Notes': ''}}])
-
-            if response['statistics']['emails'].get('values') is not None:
-                emailKeys = list(response['statistics']['emails'].get('values').keys())
-                for emailKey in emailKeys:
-                    returnResults.append([{'Email Address': str(emailKey),
-                                           'Entity Type': 'Email Address'},
-                                          {uid: {'Resolution': 'Email Address Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['countries'].get('values') is not None:
-                countriesKeys = list(response['statistics']['countries'].get('values').keys())
-                for countriesKey in countriesKeys:
-                    returnResults.append([{'Country Name': str(countriesKey),
-                                           'Entity Type': 'Country'},
-                                          {uid: {'Resolution': 'Country Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['languages'].get('values') is not None:
-                languagesKeys = list(response['statistics']['languages'].get('values').keys())
-                for languagesKey in languagesKeys:
-                    returnResults.append([{'Phrase': str(languagesKey),
-                                           'Entity Type': 'Phrase'},
-                                          {uid: {'Resolution': 'Language Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['ibans'].get('values') is not None:
-                ibansKeys = list(response['statistics']['ibans'].get('values').keys())
-                for ibansKey in ibansKeys:
-                    returnResults.append([{'Account Number': str(ibansKey),
-                                           'Entity Type': 'Bank Account'},
-                                          {uid: {'Resolution': 'IBAN Entity',
-                                                 'Notes': ''}}])
-        return returnResults

Modules/Aleph/GetCollectionsInfo.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetCollectionsInfo:
-    name = "Get Collections Info"
-    description = "Find information about Collections and their IDs"
-    originTypes = {'Phrase'}
-    resultTypes = {'Phrase, Aleph ID'}
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. ',
-                                  'type': 'String',
-                                  'default': '1'},
-                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}
-                  }
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "The value for parameter 'Max Results' is not a valid integer."
-        if linkNumbers <= 0:
-            return []
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                url = f"https://aleph.occrp.org/api/2/collections?offset=0&limit=300&page"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            max_results = int(len(response['results']))
-            if linkNumbers >= max_results:
-                collections = response['results']
-            else:
-                collections = response['results'][0: linkNumbers]
-
-            for collection in collections:
-                index_of_child = len(returnResults)
-                returnResults.append([{'Phrase': collection['label'],
-                                       'Notes': str(collection.get('summary')),
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aleph Collection Name',
-                                             'Notes': ''}}])
-
-                returnResults.append([{'ID': collection['id'],
-                                       'Entity Type': 'Aleph ID'},
-                                      {index_of_child: {'Resolution': 'Aleph Collection ID',
-                                                        'Notes': ''}}])
-        return returnResults

Modules/Aleph/GetSimilarEntities.py

@@ -1,101 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetSimilarEntities:
-    name = "Get Similar Entities"
-    description = "Find information about similar entities"
-    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
-    resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID'}
-    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        import pycountry
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                url = f"https://aleph.occrp.org/api/2/entities/{primary_field}/similar"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            entities = response['results']
-            for schema in entities:
-                if schema['entity']['schema'] == 'Person':
-                    index_of_child = len(returnResults)
-                    returnResults.append([{'Full Name': ' '.join(map(str, schema['entity']['properties']['name'])),
-                                           'Gender': ' '.join(map(str, schema['entity']['properties']['gender'])),
-                                           'Notes': ' '.join(map(str, schema['entity']['properties']['legalForm'])),
-                                           'Entity Type': 'Person'},
-                                          {uid: {'Resolution': 'Person Entity',
-                                                 'Notes': ''}}])
-                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
-                    returnResults.append([{'Street Address': schema['entity']['properties']['addressEntity'][0]
-                                           ['properties']['full'][0],
-                                           'Postal Code': schema['entity']['properties']['addressEntity'][0]
-                                           ['properties']['postalCode'][0],
-                                           'Country': country,
-                                           'Entity Type': 'Address'},
-                                          {index_of_child: {'Resolution': 'Address',
-                                                            'Notes': ''}}])
-                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
-                                           'Notes': schema['entity']['collection']['summary'],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Location in Database',
-                                                            'Notes': ''}}])
-
-                    returnResults.append([{'ID': schema['entity']['id'],
-                                           'Entity Type': 'Aleph ID'},
-                                          {index_of_child: {'Resolution': 'ID in Database',
-                                                            'Notes': ''}}])
-
-                elif schema['entity']['schema'] == 'Company':
-                    index_of_child = len(returnResults)
-                    returnResults.append([{'Company Name': ' '.join(map(str, schema['entity']['properties']['name'])),
-                                           'Notes': str(schema['entity']['properties']['status']),
-                                           'Entity Type': 'Company'},
-                                          {uid: {'Resolution': 'Company Entity',
-                                                 'Notes': ''}}])
-                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
-                    returnResults.append([{'Street Address': str(schema['entity']['properties'].get('address')),
-                                           'Country': country,
-                                           'Entity Type': 'Address'},
-                                          {index_of_child: {'Resolution': 'Address',
-                                                            'Notes': ''}}])
-                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
-                                           'Notes': schema['entity']['collection']['summary'],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Location in Database',
-                                                            'Notes': ''}}])
-
-                    returnResults.append([{'ID': schema['entity']['id'],
-                                           'Entity Type': 'Aleph ID'},
-                                          {index_of_child: {'Resolution': 'ID in Database',
-                                                            'Notes': ''}}])
-        return returnResults

Modules/Aleph/requirements.txt

@@ -1,2 +0,0 @@
-requests
-pycountry

Modules/Amass/Amass_Domain.py

@@ -1,234 +0,0 @@
-#!/usr/bin/env python3
-# User needs to be in docker group or to have root privileges
-
-class Amass_Domain:
-    name = "Amass Domain Scan"
-    description = "Find information about a particular domain"
-    originTypes = {'Domain'}
-    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain'}
-    parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://virustotal.com.',
-                                 'type': 'String',
-                                 'value': 'None',
-                                 'global': True,
-                                 'default': 'None'},
-                  'AlienVault': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://otx.alienvault.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'BinaryEdge': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://app.binaryedge.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'C99': {'description': 'Enter your api key under your profile after'
-                                         ' signing up on https://c99.nl.',
-                          'type': 'String',
-                          'value': 'None'},
-                  'Censys': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://censys.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Chaos': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://chaos.projectdiscovery.io.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'Cloudflare': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://cloudflare.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'DNSDB': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://dnsdb.info.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'GitHub': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://github.com.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Hunter': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://hunter.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'IPinfo': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://ipinfo.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'NetworksDB': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://networksdb.io.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'PassiveTotal': {'description': 'Enter your api key under your profile after'
-                                                  ' signing up on https://passivetotal.com .',
-                                   'type': 'String',
-                                   'value': 'None'},
-                  'ReconDev': {'description': 'Enter your api key under your profile after'
-                                              ' signing up on https://recon.dev.',
-                               'type': 'String',
-                               'value': 'None'},
-                  'SecurityTrails': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://securitytrails.com.',
-                                     'type': 'String',
-                                     'value': 'None'},
-                  'Shodan': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://shodan.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Spyse': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://spyse.com.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'ThreatBook': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://threatbook.cn.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'Umbrella': {'description': 'Enter your api key under your profile after'
-                                              ' signing up on https://umbrella.cisco.com.',
-                               'type': 'String',
-                               'value': 'None'},
-                  'URLScan': {'description': 'Enter your api key under your profile after'
-                                             ' signing up on https://urlscan.io.',
-                              'type': 'String',
-                              'value': 'None'},
-                  'WhoisXMLAPI': {'description': 'Enter your api key under your profile after'
-                                                 ' signing up on https://whoisxmlapi.com.',
-                                  'type': 'String',
-                                  'value': 'None'},
-                  'ZETAlytics': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://zetalytics.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'ZoomEye': {'description': 'Please Enter the Username and password with a space '
-                                             'in between',
-                              'type': 'String',
-                              'value': 'None'},
-                  'FacebookCT': {'description': 'Please Enter the api key and secret with a space '
-                                                'in between. Obtain them at https://developer.facebook.com',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'Twitter': {'description': 'Please Enter the api key and secret with a space '
-                                             'in between. Obtain them at https://developer.twitter.com',
-                              'type': 'String',
-                              'value': 'None'},
-                  'ReconDev.free': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None'},
-                  'ReconDev.paid': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        import json
-        from ipaddress import ip_address, IPv4Address, IPv6Address
-        import docker
-        import tempfile
-        from docker.errors import APIError
-
-        return_result = []
-        client = docker.from_env()
-        # Generate Config as a temporary file:
-        for entity in entityJsonList:
-            primary_field = entity[list(entity)[1]].strip()
-            try:
-                with tempfile.TemporaryDirectory() as tempDir:
-                    tempPath = Path(tempDir).absolute()
-                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                         suffix='Config',
-                                                         dir=tempPath)
-                    config.write("share = true\n")
-                    config.write("[scope]\n")
-                    config.write("port = 80\n")
-                    config.write("port = 443\n")
-                    config.write("[data_sources]\n")
-                    config.write("minimum_ttl = 1440\n")
-                    for parameter in parameters:
-                        if parameters[f'{parameter}'] != 'None':
-                            field1 = f"[data_sources.{parameter}]"
-                            field2 = f"[data_sources.{parameter}.Credentials]"
-                            if parameter == "ZoomEye":
-                                username, password = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"username = {username}\n")
-                                config.write(f"password = {password}\n")
-                            elif parameter == "FacebookCT":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.app1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "Twitter":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.account1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "ReconDev.paid":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.paid\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            elif parameter == "ReconDev.free":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.free\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            else:
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            path_to_config = config.name
-                            path_to_config = path_to_config.replace(str(tempPath), "")
-                            config.seek(0)
-                            container = client.containers.run("caffix/amass:latest",
-                                                              f"enum -src -d {primary_field} "
-                                                              f"-config /.config/amass{path_to_config}",
-                                                              volumes={
-                                                                  str(tempPath): {'bind': '/.config/amass',
-                                                                                  'mode': 'rw'}},
-                                                              remove=True)
-                        else:
-                            container = client.containers.run("caffix/amass:latest",
-                                                              f"enum -src -d {primary_field}",
-                                                              volumes={
-                                                                  str(tempPath): {'bind': '/.config/amass',
-                                                                                  'mode': 'rw'}},
-                                                              remove=True)
-                    jsonFile = tempPath / 'amass.json'
-                    if jsonFile.exists():
-                        jsonContents = ""
-                        with open(jsonFile, 'r') as jsonFileHandler:
-                            jsonContents = jsonFileHandler.read()
-            except (APIError, docker.errors.ContainerError) as error:
-                return "Something happened to docker - Cannot continue"
-            uid = entity['uid']
-            for dictionary in jsonContents.splitlines():
-                index_of_child = len(return_result)
-                line_dictionary = json.loads(dictionary)
-                size = len(line_dictionary['addresses'])
-                return_result.append([{'Domain Name': str(line_dictionary['name']),
-                                       'Entity Type': 'Domain'},
-                                      {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
-                for ip in range(size):
-                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
-                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IP Address'},
-                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
-                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
-                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IPv6 Address'},
-                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
-                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
-                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
-                                           'Entity Type': 'Autonomous System'},
-                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
-                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Amass Domain Scan Description',
-                                                            'Notes': ''}}])
-        jsonFileHandler.close()
-        return return_result

Modules/Amass/Amass_Intel.py

@@ -1,293 +0,0 @@
-#!/usr/bin/env python3
-# User needs to be in docker group or have root privileges
-
-class Amass_Intel:
-    name = "Amass Intel Scan"
-    description = "Find information about a particular domain"
-    originTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
-    resultTypes = {'Domain', 'IP Address', 'Autonomous System', 'Phrase', 'Company', 'Organization'}
-    parameters = {'VirusTotal': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://virustotal.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'AlienVault': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://otx.alienvault.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'BinaryEdge': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://app.binaryedge.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'C99': {'description': 'Enter your api key under your profile after'
-                                         ' signing up on https://c99.nl.',
-                          'type': 'String',
-                          'value': 'None'},
-                  'Censys': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://censys.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Chaos': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://chaos.projectdiscovery.io.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'Cloudflare': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://cloudflare.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'DNSDB': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://dnsdb.info.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'GitHub': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://github.com.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Hunter': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://hunter.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'IPinfo': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://ipinfo.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'NetworksDB': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://networksdb.io.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'PassiveTotal': {'description': 'Enter your api key under your profile after'
-                                                  ' signing up on https://passivetotal.com .',
-                                   'type': 'String',
-                                   'value': 'None'},
-                  'ReconDev': {'description': 'Enter your api key under your profile after'
-                                              ' signing up on https://recon.dev.',
-                               'type': 'String',
-                               'value': 'None'},
-                  'SecurityTrails': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://securitytrails.com.',
-                                     'type': 'String',
-                                     'value': 'None'},
-                  'Shodan': {'description': 'Enter your api key under your profile after'
-                                            ' signing up on https://shodan.io.',
-                             'type': 'String',
-                             'value': 'None'},
-                  'Spyse': {'description': 'Enter your api key under your profile after'
-                                           ' signing up on https://spyse.com.',
-                            'type': 'String',
-                            'value': 'None'},
-                  'ThreatBook': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://threatbook.cn.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'Umbrella': {'description': 'Enter your api key under your profile after'
-                                              ' signing up on https://umbrella.cisco.com.',
-                               'type': 'String',
-                               'value': 'None'},
-                  'URLScan': {'description': 'Enter your api key under your profile after'
-                                             ' signing up on https://urlscan.io.',
-                              'type': 'String',
-                              'value': 'None'},
-                  'WhoisXMLAPI': {'description': 'Enter your api key under your profile after'
-                                                 ' signing up on https://whoisxmlapi.com.',
-                                  'type': 'String',
-                                  'value': 'None'},
-                  'ZETAlytics': {'description': 'Enter your api key under your profile after'
-                                                ' signing up on https://zetalytics.com.',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'ZoomEye': {'description': 'Please Enter the Username and password with a space '
-                                             'in between',
-                              'type': 'String',
-                              'value': 'None'},
-                  'FacebookCT': {'description': 'Please Enter the api key and secret with a space '
-                                                'in between. Obtain them at https://developer.facebook.com',
-                                 'type': 'String',
-                                 'value': 'None'},
-                  'Twitter': {'description': 'Please Enter the api key and secret with a space '
-                                             'in between. Obtain them at https://developer.twitter.com',
-                              'type': 'String',
-                              'value': 'None'},
-                  'ReconDev.free': {
-                      'description': 'Please Enter the api key under your"\
-                       "profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None'},
-                  'ReconDev.paid': {
-                      'description': 'Please Enter the api key under your profile"\
-                       "after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        import json
-        from ipaddress import ip_address, IPv4Address, IPv6Address
-        import docker
-        import tempfile
-        from docker.errors import APIError
-
-        return_result = []
-        client = docker.from_env()
-        # Generate Config as a temporary file:
-        for entity in entityJsonList:
-            primary_field = entity[list(entity)[1]].strip()
-            try:
-                with tempfile.TemporaryDirectory() as tempDir:
-                    tempPath = Path(tempDir).absolute()
-                    config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                         suffix='Config',
-                                                         dir=tempPath)
-                    config.write("share = true\n")
-                    config.write("[scope]\n")
-                    config.write("port = 80\n")
-                    config.write("port = 443\n")
-                    config.write("[data_sources]\n")
-                    config.write("minimum_ttl = 1440\n")
-                    for parameter in parameters:
-                        if parameters[f'{parameter}'] != 'None':
-                            field1 = f"[data_sources.{parameter}]"
-                            field2 = f"[data_sources.{parameter}.Credentials]"
-                            if parameter == "ZoomEye":
-                                username, password = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"username = {username}\n")
-                                config.write(f"password = {password}\n")
-                            elif parameter == "FacebookCT":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.app1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "Twitter":
-                                field3, secret = parameters[parameter].split(' ', 1)
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.account1\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                                config.write(f"secret = {secret}\n")
-                            elif parameter == "ReconDev.paid":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.paid\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            elif parameter == "ReconDev.free":
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"[data_sources.{parameter}.free\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            else:
-                                field3 = parameters[f'{parameter}']
-                                config.write(f"{field1}\n")
-                                config.write(f"{field2}\n")
-                                config.write(f"apikey = \"{field3}\"\n")
-                            path_to_config = config.name
-                            path_to_config = path_to_config.replace(str(tempPath), "")
-                            config.seek(0)
-                            print(config.read())
-                            print(path_to_config)
-                            print(entity['Entity Type'])
-                            if entity['Entity Type'] == "Domain":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -d {primary_field} -config /.config/amass"
-                                                                  f"{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
-                                    'Entity Type'] == "Company":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -org {primary_field} -whois -config "
-                                                                  f"/.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "IP Address":
-                                try:
-                                    ip_address(primary_field)
-                                except ValueError:
-                                    return "The Entity Provided isn't a valid IP Address"
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -addr {primary_field} -whois -config "
-                                                                  f"/.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Autonomous System":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}   "
-                                                                  f" -whois -config /.config/amass{path_to_config}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                        else:
-                            if entity['Entity Type'] == "Domain":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -d {primary_field}",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Organization" or entity['Entity Type'] == "Phrase" or entity[
-                                    'Entity Type'] == "Company":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -org {primary_field} -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "IP Address":
-                                try:
-                                    ip_address(primary_field)
-                                except ValueError:
-                                    return "The Entity Provided isn't a valid IP Address"
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -addr {primary_field} -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                            elif entity['Entity Type'] == "Autonomous System":
-                                container = client.containers.run("caffix/amass:latest",
-                                                                  f"intel -src -asn {entity[list(entity)[2]].strip()}"
-                                                                  f" -whois",
-                                                                  volumes={
-                                                                      str(tempPath): {'bind': '/.config/amass',
-                                                                                      'mode': 'rw'}},
-                                                                  remove=True)
-                    jsonFile = tempPath / 'amass.json'
-                    if jsonFile.exists():
-                        jsonContents = ""
-                        with open(jsonFile, 'r') as jsonFileHandler:
-                            jsonContents = jsonFileHandler.read()
-            except (APIError, docker.errors.ContainerError) as error:
-                return "Soomething happened to docker - Cannot continue."
-            uid = entity['uid']
-            for dictionary in jsonContents.splitlines():
-                index_of_child = len(return_result)
-                line_dictionary = json.loads(dictionary)
-                size = len(line_dictionary['addresses'])
-                return_result.append([{'Domain Name': str(line_dictionary['name']),
-                                       'Entity Type': 'Domain'},
-                                      {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
-                for ip in range(size):
-                    if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
-                        return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IP Address'},
-                                              {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
-                    elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
-                        return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
-                                               'Entity Type': 'IPv6 Address'},
-                                              {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
-                    return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
-                                           'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
-                                           'Entity Type': 'Autonomous System'},
-                                          {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
-                    return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Amass Intel Scan Description',
-                                                            'Notes': ''}}])
-        jsonFileHandler.close()
-        return return_result

Modules/Amass/requirements.txt

@@ -1 +0,0 @@
-docker

Modules/BigMatch/BigMatch.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BigMatch:
-    name = "BigMatch Search"
-    description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
-    originTypes = {"Image", "Document", "Archive"}
-    resultTypes = {'Website'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        from playwright.sync_api import sync_playwright, TimeoutError
-        from bs4 import BeautifulSoup
-
-        return_result = []
-
-        url = "https://bigmatch.rev.ng/static/index.html"
-        failString = 'Too many strings in binary?'
-        successString = 'Results:'
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-
-            for entity in entityJsonList:
-                uid = entity['uid']
-                file_path = Path(parameters['Project Files Directory']) / entity["File Path"]
-                file_path = file_path.absolute()
-                if not (file_path.exists() and file_path.is_file()):
-                    continue
-                page.wait_for_timeout(3000)
-
-                for _ in range(3):
-                    try:
-                        page.goto(url, wait_until="networkidle", timeout=10000)
-                        inputLocator = page.locator("input")
-                        inputLocator.set_input_files([str(file_path)])
-                        page.wait_for_timeout(3000)
-                        soup = BeautifulSoup(page.content(), 'lxml')
-                        soupText = soup.get_text()
-                        while (failString not in soupText) and (successString not in soupText):
-                            page.wait_for_timeout(1000)
-                            soup = BeautifulSoup(page.content(), 'lxml')
-                            soupText = soup.get_text()
-                        if failString in soupText:
-                            return []
-                        for link in soup.find_all('a'):
-                            potentialLink = link.get('href', None)
-                            if potentialLink is not None:
-                                if 'github' in potentialLink:
-                                    return_result.append([{'URL': potentialLink, 'Entity Type': 'Website'},
-                                                          {uid: {'Resolution': 'BigMatch Github Link', 'Notes': ''}}])
-
-                        break
-                    except TimeoutError:
-                        pass
-            page.close()
-            browser.close()
-        return return_result

Modules/BigMatch/requirements.txt

@@ -1 +0,0 @@
-beautifulsoup4

Modules/CryptoCurrency/BlockChainAPI.py

@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainAPI:
-    # A string that is treated as the name of this resolution.
-    name = "BlockChainAPI Get Transaction Details"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes of balance, sent, received info"
-
-    originTypes = {'Crypto Wallet', 'Phrase'}
-
-    resultTypes = {'Currency'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import datetime
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            search_address = entity[list(entity)[1]]
-
-            try:
-                getSentByAddress = requests.get(
-                    f'https://blockchain.info/q/getsentbyaddress/{search_address}?confirmations=6')
-                if getSentByAddress.status_code == 404:
-                    continue
-                time.sleep(1)
-                getReceivedByAddress = requests.get(
-                    f'https://blockchain.info/q/getreceivedbyaddress/{search_address}?confirmations=6')
-                time.sleep(1)
-                addressBalance = requests.get(
-                    f'https://blockchain.info/q/addressbalance/{search_address}?confirmations=6')
-                time.sleep(1)
-                addressFirstSeen = requests.get(
-                    f'https://blockchain.info/q/addressfirstseen/{search_address}?confirmations=6')
-                time.sleep(1)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            dateCreated = str(datetime.datetime.fromtimestamp(float(addressFirstSeen.text))).replace(" ", 'T')
-
-            returnResults.append(
-                [{'Amount': str(int(getSentByAddress.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'sent by address',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'sent by address',
-                        'Name': 'sent by address',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Amount': str(int(getReceivedByAddress.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'received by address',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'received by address',
-                        'Name': 'received by address',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Amount': str(int(addressBalance.text) / 100000000),
-                  'Currency Type': 'BTC',
-                  'Notes': 'address balance',
-                  'Entity Type': 'Currency'},
-                 {uid: {'Resolution': 'address balance',
-                        'Name': 'address balance',
-                        'Notes': ''}}])
-            returnResults.append(
-                [{'Date': str(dateCreated),
-                  'Notes': 'first seen date',
-                  'Entity Type': 'Date'},
-                 {uid: {'Resolution': 'first seen date',
-                        'Name': 'first seen date',
-                        'Notes': ''}}])
-            return returnResults

Modules/CryptoCurrency/CryptoCurrency.xml

@@ -1,7 +0,0 @@
-<CryptoCurrency>
-  <Crypto_Wallet>
-    <Attributes>
-      Wallet Address
-    </Attributes>
-  </Crypto_Wallet>
-</CryptoCurrency>

Modules/CryptoCurrency/EtherScanGetBalance.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EtherScanGetBalance:
-    name = "EtherScan.io Get Balance"
-    description = "EtherScan get the balance of the selected account"
-    originTypes = {"Crypto Wallet"}
-    resultTypes = {'Crypto Wallet'}
-    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io",
-                                        'type': 'String',
-                                        'value': '',
-                                        'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-
-        return_result = []
-
-        api_key = parameters['EtherScan API Key']
-        url = "https://api.etherscan.io/api"
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primary_field = entity[list(entity)[1]]
-            crafted_url = f"{url}?module=account&action=balance&address={primary_field}&tag=latest&apikey={api_key}"
-            try:
-                response = requests.get(crafted_url)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            response = response.json()
-            return_result.append([{'Amount': response['result'],
-                                   'Currency': 'Ethereum',
-                                   'Entity Type': 'Currency'},
-                                  {uid: {'Resolution': 'EtherScan.io Account Balance', 'Notes': ''}}])
-        return return_result

Modules/CryptoCurrency/EtherScanGetBlocksMined.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EtherScanGetBlocksMined:
-    name = "EtherScan.io Get Blocks Mined"
-    description = "EtherScan Blocks Mined from the selected account"
-    originTypes = {"Crypto Wallet"}
-    resultTypes = {'Crypto Wallet'}
-    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io",
-                                        'type': 'String',
-                                        'value': '',
-                  'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-
-        return_result = []
-
-        api_key = parameters['EtherScan API Key']
-        url = "https://api.etherscan.io/api"
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primary_field = entity[list(entity)[1]]
-            crafted_url =\
-                f"{url}?module=account&action=getminedblocks&address={primary_field}&tag=latest&apikey={api_key}"
-            try:
-                response = requests.get(crafted_url)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            response = response.json()
-            return_result.append([{'Phrase': response['result'],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'EtherScan.io Blocks Mined', 'Notes': ''}}])
-        return return_result

Modules/CryptoCurrency/requirements.txt

@@ -1 +0,0 @@
-requests

Modules/EDGAR_&_EFD/CompanyInfo.py

@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-
-class CompanyInfo:
-    # A string that is treated as the name of this resolution.
-    name = "Get Company Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes containing Company Information"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Phrase, SIC, EIN, Address'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
-        }
-
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(1)
-            search_url = f'https://data.sec.gov/submissions/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            time.sleep(1)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-
-            exchanges = data['exchanges']
-            for exchange in exchanges:
-                returnResults.append([{'Exchange Name': exchange,
-                                       'Entity Type': 'Exchanges'},
-                                      {uid: {'Resolution': 'Exchange',
-                                             'Name': 'Exchange',
-                                             'Notes': ''}}])
-
-            tickers = data['tickers']
-            for ticker in tickers:
-                returnResults.append([{'Ticker Name': ticker,
-                                       'Entity Type': 'Tickers'},
-                                      {uid: {'Resolution': 'Exchange',
-                                             'Name': 'Exchange',
-                                             'Notes': ''}}])
-
-            if data['insiderTransactionForOwnerExists'] == 1:
-                returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-            else:
-                returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-
-            if data['insiderTransactionForIssuerExists'] == 1:
-                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-            else:
-                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-
-            if data['sic'] is not None:
-                returnResults.append([{'SIC': str(data['sic']),
-                                       'Description': data['sicDescription'],
-                                       'Entity Type': 'SIC'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-            if data['ein'] is not None:
-                returnResults.append([{'EIN': str(data['ein']),
-                                       'Entity Type': 'EIN'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-            if data['addresses'] is not None:
-                returnResults.append([{'Street Address': data['addresses']['mailing']['street1'],
-                                       'Postal Code': data['addresses']['mailing']['zipCode'],
-                                       'Country': data['addresses']['mailing']['stateOrCountry'],
-                                       'Locality': data['addresses']['mailing']['city'],
-                                       'Entity Type': 'Address'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-
-            if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']:
-                returnResults.append([{'Street Address': data['addresses']['business']['street1'],
-                                       'Postal Code': data['addresses']['business']['zipCode'],
-                                       'Country': data['addresses']['business']['stateOrCountry'],
-                                       'Locality': data['addresses']['business']['city'],
-                                       'Entity Type': 'Address'},
-                                      {uid: {'Resolution': '',
-                                             'Name': '',
-                                             'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/CompanyToCIK.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-
-class CompanyToCIK:
-    # A string that is treated as the name of this resolution.
-    name = "Get CIK ID From Company"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes of contact info for websites"
-
-    originTypes = {'Phrase', 'Company'}
-
-    resultTypes = {'Phrase'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import re
-        from bs4 import BeautifulSoup
-        from playwright.sync_api import sync_playwright, TimeoutError
-
-        returnResults = []
-        index_of_child = []
-        cikRegex = re.compile(r'CIK=\d{4,10}', re.IGNORECASE)
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-            for entity in entityJsonList:
-                page.wait_for_timeout(1000)
-                uid = entity['uid']
-                search_term = entity[list(entity)[1]]
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(f'https://www.sec.gov/cgi-bin/browse-edgar?company={search_term}',
-                                  wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                if not pageResolved:
-                    continue
-
-                soup = BeautifulSoup(page.content(), 'lxml')
-                cikIDs = cikRegex.findall(soup.get_text())
-
-                links_with_text = []
-                for td_element in soup.find_all('td'):
-                    if td_element.text:
-                        try:
-                            text = td_element.text
-                            split = text.split('SIC')[0]
-                            links_with_text.append(split)
-                        except IndexError:
-                            links_with_text.append(td_element.text)
-
-                for link in links_with_text:
-                    if search_term.lower() in link.lower():
-                        index_of_child.append(len(returnResults))
-                        returnResults.append([{'Company Name': link,
-                                               'Entity Type': 'Company'},
-                                              {uid: {'Resolution': 'Edgar Company',
-                                                     'Notes': ''}}])
-                for code in cikIDs:
-                    returnResults.append([{'CIK': code.split('=')[1],
-                                           'Entity Type': 'Edgar ID'},
-                                          {index_of_child[cikIDs.index(code)]: {'Resolution': 'CIK Edgar ID',
-                                                                                'Notes': ''}}])
-            page.close()
-            browser.close()
-        return returnResults