Small changes to networking to handle unexpected fragmentation.

The expected network traffic of the tool might increase slightly - these changes will be looked at in the future to see if it is possible to cut down on the traffic.

Core/EntityDB.py

@@ -159,7 +159,7 @@ class EntitiesDB:
             returnValue = self.database.nodes[uid]
         except KeyError:
             self.messageHandler.warning(
-                "Tried to get entity with nonexistent UID.")
+                "Tried to get entity with nonexistent UID: " + uid)
         finally:
             self.dbLock.release()
             return returnValue
@@ -366,7 +366,7 @@ class EntitiesDB:
             self.dbLock.release()
             return returnValue
 
-    def mergeDatabases(self, newDB: nx.DiGraph, fromServer=True):
+    def mergeDatabases(self, newDB_nodes: dict, newDB_edges: dict, fromServer=True):
         """
         Merges the existing database with the one provided.
         
@@ -375,29 +375,30 @@ class EntitiesDB:
         self.dbLock.acquire()
         differenceGraph = nx.DiGraph()
         # Note: If we ever receive a node without a 'Date Last Edited' field, ignore it.
-        differenceGraph.add_nodes_from([(x, newDB.nodes[x])
-                                        for x in newDB.nodes() if (x not in self.database.nodes()) or
+        differenceGraph.add_nodes_from([(x, newDB_nodes[x])
+                                        for x in newDB_nodes if (x not in self.database.nodes()) or
                                         (
                                                 x in self.database.nodes() and
-                                                newDB.nodes[x].get('Date Last Edited') and
-                                                newDB.nodes[x]['Date Last Edited'] >
+                                                newDB_nodes[x].get('Date Last Edited') and
+                                                newDB_nodes[x]['Date Last Edited'] >
                                                 self.database.nodes[x]['Date Last Edited']
                                         )
                                         ])
-        differenceGraph.add_edges_from([(x, y, newDB.edges[(x, y)])
-                                        for x, y in newDB.edges() if ((x, y) not in self.database.edges()) or
+        differenceGraph.add_edges_from([(x, y, newDB_edges[(x, y)])
+                                        for x, y in newDB_edges if ((x, y) not in self.database.edges()) or
                                         (
                                                 (x, y) in self.database.edges() and
-                                                newDB.edges[(x, y)].get('Date Last Edited') and
-                                                newDB.edges[(x, y)]['Date Last Edited'] >
+                                                newDB_edges[(x, y)].get('Date Last Edited') and
+                                                newDB_edges[(x, y)]['Date Last Edited'] >
                                                 self.database.edges[(x, y)]['Date Last Edited']
                                         )
                                         ])
         if differenceGraph.number_of_nodes() > 0:
-            for node in differenceGraph.nodes:
-                print('Merging Node: ' + str(node))
-                self.mainWindow.populateEntitiesWidget(differenceGraph.nodes[node], add=True)
             self.database = nx.compose(self.database, differenceGraph)
+            # Some nodes given by differenceGraph may be empty dicts, with an existing node's uid as the key.
+            for node in differenceGraph.nodes:
+                self.mainWindow.populateEntitiesWidget(self.database.nodes[node], add=True)
+
             if not fromServer:
                 if self.mainWindow.FCOM.isConnected():
                     # diffNew = nx.DiGraph()

Core/FrontendCommunicationsHandler.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
+
+from ast import literal_eval
 from typing import Union
 from pathlib import Path
 from msgpack import loads, dumps
 import socket
 import threading
 import time
-import pickle
 import re
 import networkx as nx
 
@@ -54,13 +55,14 @@ class CommunicationsHandler(QtCore.QObject):
     open_project_canvas_signal = QtCore.Signal(str)
     close_project_canvas_signal = QtCore.Signal(str)
     receive_project_database_update = QtCore.Signal(dict, bool)
-    receive_project_canvas_update = QtCore.Signal(str, str)
-    receive_sync_database = QtCore.Signal(nx.DiGraph)
+    receive_project_canvas_update_node = QtCore.Signal(str, str)
+    receive_project_canvas_update_link = QtCore.Signal(str, tuple)
+    receive_sync_database = QtCore.Signal(dict, dict)
     status_message_signal = QtCore.Signal(str, bool)
     receive_project_file_list = QtCore.Signal(list)
     file_upload_finished_signal = QtCore.Signal(str)
     file_upload_abort_signal = QtCore.Signal(str)
-    receive_sync_canvas_signal = QtCore.Signal(str, nx.DiGraph)
+    receive_sync_canvas_signal = QtCore.Signal(str, dict, dict)
 
     def __init__(self, mainWindow):
 
@@ -87,7 +89,8 @@ class CommunicationsHandler(QtCore.QObject):
         self.close_project_signal.connect(self.mainWindow.closeCurrentServerProject)
         self.close_project_canvas_signal.connect(self.mainWindow.closeServerCanvasListener)
         self.receive_project_database_update.connect(self.mainWindow.receiveServerDatabaseUpdate)
-        self.receive_project_canvas_update.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_project_canvas_update_node.connect(self.mainWindow.receiveServerCanvasUpdate)
+        self.receive_project_canvas_update_link.connect(self.mainWindow.receiveServerCanvasUpdate)
         self.receive_sync_database.connect(self.mainWindow.receiveSyncDatabaseListener)
         self.receive_project_file_list.connect(self.mainWindow.receiveFileListListener)
         self.file_upload_finished_signal.connect(self.mainWindow.fileUploadFinishedListener)
@@ -190,7 +193,7 @@ class CommunicationsHandler(QtCore.QObject):
         #   sent will always be less than or equal to 1280 bytes.
         padNeeded = 1280 - len(bytesObject)
         message = encryptor.update(b'a' * padNeeded + bytesObject) + encryptor.finalize()
-        return message
+        return b64encode(message) + b'\x00\x00\x00'
 
     def decryptTransmission(self, bytesObject):
         decrypter = self.cipher.decryptor()
@@ -205,7 +208,7 @@ class CommunicationsHandler(QtCore.QObject):
         # Note that Base64 encoded data is about 4/3 times the size of the original.
         # 768 * 4/3 = 1024
         print('Sending message:', messageJson)
-        argEncoded = b64encode(pickle.dumps(messageJson))
+        argEncoded = b64encode(str(messageJson).encode())
         largeMessageUUID = str(uuid4())
         try:
             for data in range(0, len(argEncoded), 768):
@@ -249,14 +252,23 @@ class CommunicationsHandler(QtCore.QObject):
         it exists.
         """
         preInbox = {}
+        oldData = b''
         while True:
             try:
                 receivedInfo = self.sock.recv(5120)
                 if receivedInfo == b'':
                     # Socket closed.
                     break
-                for message in range(0, len(receivedInfo), 1280):
-                    decryptedInfo = self.decryptTransmission(receivedInfo[message:message + 1280])
+                receivedInfo = oldData + receivedInfo
+                messages = receivedInfo.split(b'\x00\x00\x00')
+                if not receivedInfo.endswith(b'\x00\x00\x00'):
+                    oldData = messages[-1]
+                    messages = messages[:-1]
+                else:
+                    oldData = b''
+                for message in messages:
+                    message = b64decode(message)
+                    decryptedInfo = self.decryptTransmission(message)
                     if decryptedInfo is None:
                         self.mainWindow.MESSAGEHANDLER.warning("Invalid message received from server.")
                         continue
@@ -267,7 +279,8 @@ class CommunicationsHandler(QtCore.QObject):
                     else:
                         preInbox[messageID] = receivedMessage
                     if receivedMessage.get("done"):
-                        preInbox[messageID]["message"] = pickle.loads(b64decode(preInbox[messageID]["message"]))
+                        preInbox[messageID]["message"] = literal_eval(
+                            b64decode(preInbox[messageID]["message"]).decode())
                         self.inbox.put(preInbox.pop(messageID).get("message"))
 
             except socket.error as socketError:
@@ -282,15 +295,9 @@ class CommunicationsHandler(QtCore.QObject):
                     break
             except ValueError:
                 # E.g.: Unpack failed: incomplete input
-                pass
-            except pickle.UnpicklingError:
-                # If something went wrong with one of the packets, ignore.
-                # User can re-sync the database if needed.
-                # This rarely occurs in normal operation.
-                pass
-            except ModuleNotFoundError:
-                # Missing module to unpickle message.
-                pass
+                # In this case, we are being sent fragmented messages.
+                # They will be reconstructed eventually, once all the pieces get here.
+                continue
 
     def askServerForResolutions(self):
         message = {"Operation": "Get Server Resolutions",
@@ -302,10 +309,20 @@ class CommunicationsHandler(QtCore.QObject):
 
     def runRemoteResolution(self, resolution_name: str, resolution_entities: list, resolution_parameters: dict,
                             resolution_uid: str):
+        resolution_entities_to_send = []
+        for entity in resolution_entities:
+            try:
+                dereferenced_entity = dict(entity)
+                resolution_entities_to_send.append(dereferenced_entity)
+                # Icon is not necessary for any resolution as of now: 2022/1/2.
+                # Cutting it out saves data.
+                dereferenced_entity['Icon'] = ''
+            except KeyError:
+                pass
         message = {'Operation': 'Run Resolution',
                    'Arguments': {
                        'resolution_name': resolution_name,
-                       'resolution_entities': resolution_entities,
+                       'resolution_entities': resolution_entities_to_send,
                        'resolution_parameters': resolution_parameters,
                        'resolution_uid': resolution_uid
                    }}
@@ -390,12 +407,14 @@ class CommunicationsHandler(QtCore.QObject):
         message = {'Operation': 'Sync Database',
                    'Arguments': {
                        'project_name': project_name,
-                       'client_project_graph': client_project_graph
+                       'client_project_graph': str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(
+                           client_project_graph))
                    }}
         self.transmitMessage(message)
 
-    def receiveSyncDatabase(self, database: nx.DiGraph):
-        self.receive_sync_database.emit(database)
+    def receiveSyncDatabase(self, database: str):
+        database_nodes, database_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(database)
+        self.receive_sync_database.emit(database_nodes, database_edges)
 
     def askServerForFileList(self, project_name: str):
         message = {"Operation": "Get File List",
@@ -408,11 +427,12 @@ class CommunicationsHandler(QtCore.QObject):
                        'Arguments': {
                            'project_name': project_name,
                            'canvas_name': canvas_name,
-                           "canvas_graph": canvas_graph}}
+                           "canvas_graph": str(self.mainWindow.RESOURCEHANDLER.deconstructGraph(canvas_graph))}}
             self.transmitMessage(message)
 
-    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: nx.DiGraph):
-        self.receive_sync_canvas_signal.emit(canvas_name, canvas_graph)
+    def receiveSyncCanvas(self, canvas_name: str, canvas_graph: str):
+        graph_nodes, graph_edges = self.mainWindow.RESOURCEHANDLER.reconstructGraph(canvas_graph)
+        self.receive_sync_canvas_signal.emit(canvas_name, graph_nodes, graph_edges)
 
     def closeCanvas(self, project_name: str, canvas_name: str):
         if self.isConnected():
@@ -429,9 +449,17 @@ class CommunicationsHandler(QtCore.QObject):
     # Being verbose is better than prematurely optimizing for a few kbps of
     #   network traffic.
     def receiveDatabaseUpdateEvent(self, entity_json: dict, add: bool):
+        try:
+            entity_json['Icon'] = QtCore.QByteArray(b64decode(entity_json['Icon']))
+        except KeyError:
+            pass
         self.receive_project_database_update.emit(entity_json, add)
 
     def sendDatabaseUpdateEvent(self, project_name: str, entity_json: dict, add: bool):
+        try:
+            entity_json['Icon'] = entity_json['Icon'].toBase64().data()
+        except KeyError:
+            pass
         message = {"Operation": "Update Project Entities",
                    "Arguments": {
                        'project_name': project_name,
@@ -439,8 +467,11 @@ class CommunicationsHandler(QtCore.QObject):
                        "add": add}}
         self.transmitMessage(message)
 
-    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: str):
-        self.receive_project_canvas_update.emit(canvas_name, entity_or_link_uid)
+    def receiveCanvasUpdateEvent(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
+        if isinstance(entity_or_link_uid, str):
+            self.receive_project_canvas_update_node.emit(canvas_name, entity_or_link_uid)
+        else:
+            self.receive_project_canvas_update_link.emit(canvas_name, entity_or_link_uid)
 
     def sendCanvasUpdateEvent(self, project_name: str, canvas_name: str, entity_or_link_uid: Union[str, tuple]):
         message = {"Operation": "Update Canvas Entities",
@@ -519,16 +550,20 @@ class CommunicationsHandler(QtCore.QObject):
         This function checks if there is anything in the inbox, and if
         there is, calls the appropriate functions.
         """
+        prevMesg = None
         while True:
             try:
-                message = self.inbox.get(timeout=0.1)
+                message = self.inbox.get(timeout=0.2)
             except Empty:
                 with closeSoftwareLock:
                     if not closeSoftware:
-                        time.sleep(0.1)
+                        time.sleep(0.2)
                         continue
                     else:
                         return
+            if prevMesg == message:
+                # Same message, do not waste time handling.
+                continue
             print('Message To handle:', message)  # TODO Make this logging.
             operation = message['Operation']
             arguments = message['Arguments']
@@ -567,6 +602,7 @@ class CommunicationsHandler(QtCore.QObject):
             else:
                 self.mainWindow.MESSAGEHANDLER.warning('Unhandled message: ' + str(message) +
                                                        ' On Operation: ' + str(operation))
+            prevMesg = message
 
     def handleStatusMessage(self, operation: str, message: str, status_code: int):
         """

Core/Interface/CentralPane.py

@@ -90,6 +90,10 @@ class TabBar(QtWidgets.QTabBar):
                 return
 
             if delete:
+                sceneToClose = self.parent().canvasTabs[currName].scene()
+                groupNodes = [item for item in sceneToClose.items() if isinstance(item, Entity.GroupNode)]
+                for groupNode in groupNodes:
+                    sceneToClose.removeNode(groupNode)
                 tabIndex = self.parent().getTabIndexByName(currName)
                 self.parent().closeTab(tabIndex)
                 return
@@ -313,7 +317,13 @@ class TabbedPane(QtWidgets.QTabWidget):
         # Get all the entities, then split it into several lists, to make searching & iterating through them faster.
         allEntities = [(entity['uid'], (entity[list(entity)[1]], entity['Entity Type']))
                        for entity in self.entityDB.getAllEntities()]
-        allEntityUIDs, allEntityPrimaryFieldsAndTypes = map(list, zip(*allEntities))
+        # In case we have no entities in the database when the resolution finishes, i.e. the user deletes the origin
+        #   node for the resolution, or runs something that creates nodes from nothing.
+        if allEntities:
+            allEntityUIDs, allEntityPrimaryFieldsAndTypes = map(list, zip(*allEntities))
+        else:
+            allEntityUIDs = []
+            allEntityPrimaryFieldsAndTypes = []
         allLinks = [linkUID['uid'] for linkUID in self.entityDB.getAllLinks()]
         links = []
         newNodeUIDs = []
@@ -368,19 +378,21 @@ class TabbedPane(QtWidgets.QTabWidget):
                 parentUID = parentID
                 if isinstance(parentUID, int):
                     parentUID = newNodeUIDs[parentUID]
-                resolutionName = parentsDict[parentID]['Resolution']
-                newLinkUID = (parentUID, outputEntityUID)
-                # Avoid creating more links between the same two entities.
-                if newLinkUID in allLinks:
-                    linkJson = self.entityDB.getLinkIfExists(newLinkUID)
-                    if resolutionName not in linkJson['Notes']:
-                        linkJson['Notes'] += '\nConnection also produced by Resolution: ' + resolutionName
-                        self.entityDB.addLink(linkJson, fromServer=True)
-                else:
-                    self.entityDB.addLink({'uid': newLinkUID, 'Resolution': resolutionName,
-                                           'Notes': parentsDict[parentID]['Notes']}, fromServer=True)
-                    links.append((parentUID, outputEntityUID, resolutionName))
-                    allLinks.append(newLinkUID)
+                # Sanity check: Check that the node that was used for this resolution still exists.
+                if parentUID in allEntityUIDs:
+                    resolutionName = parentsDict[parentID]['Resolution']
+                    newLinkUID = (parentUID, outputEntityUID)
+                    # Avoid creating more links between the same two entities.
+                    if newLinkUID in allLinks:
+                        linkJson = self.entityDB.getLinkIfExists(newLinkUID)
+                        if resolutionName not in linkJson['Notes']:
+                            linkJson['Notes'] += '\nConnection also produced by Resolution: ' + resolutionName
+                            self.entityDB.addLink(linkJson, fromServer=True)
+                    else:
+                        self.entityDB.addLink({'uid': newLinkUID, 'Resolution': resolutionName,
+                                               'Notes': parentsDict[parentID]['Notes']}, fromServer=True)
+                        links.append((parentUID, outputEntityUID, resolutionName))
+                        allLinks.append(newLinkUID)
 
         progress.setValue(2)
 
@@ -441,8 +453,7 @@ class TabbedPane(QtWidgets.QTabWidget):
                             nodePrimaryAttribute = ''
                         newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
                         scene.addNodeToScene(newNode)
-                        # No need to send this link to server - it will be created automatically.
-                        scene.addLinkDragDrop(scene.nodesDict[parentUID], newNode, newLink[2], fromServer=True)
+                        scene.addLinkDragDrop(scene.nodesDict[parentUID], newNode, newLink[2])
 
                         addedNodes.append(newNode)
                     elif parentUID in scene.nodesDict and uid in scene.nodesDict:
@@ -881,19 +892,44 @@ class CanvasView(QtWidgets.QGraphicsView):
 
     def sendEntitiesToOtherCanvas(self) -> None:
         entitiesToSend = [item.uid for item in self.scene().selectedItems()
-                          if isinstance(item, Entity.BaseNode)]
+                          if isinstance(item, Entity.BaseNode) and not isinstance(item, Entity.GroupNode)]
+        groupEntitiesToSend = [item.uid for item in self.scene().selectedItems()
+                               if isinstance(item, Entity.GroupNode)]
 
         otherTabs = [tabName for tabName in self.tabbedPane.canvasTabs if tabName != self.name]
         prompt = SendToOtherTabCanvasSelector(otherTabs)
-        returnCode = prompt.exec()
 
-        if returnCode:
+        if prompt.exec():
             otherCanvasName = prompt.canvasNameSelector.currentText()
-            for entityToSend in entitiesToSend:
-                if self.tabbedPane.canvasTabs[otherCanvasName].scene().sceneGraph.nodes.get(entityToSend) is None:
-                    self.tabbedPane.canvasTabs[otherCanvasName].scene().addNodeProgrammatic(entityToSend)
+            if otherCanvasName:
+                otherCanvas = self.tabbedPane.canvasTabs[otherCanvasName].scene()
+                for entityToSend in entitiesToSend:
+                    if otherCanvas.sceneGraph.nodes.get(entityToSend) is None:
+                        otherCanvas.addNodeProgrammatic(entityToSend)
+                for groupEntityToSend in groupEntitiesToSend:
+                    # Have to make a new group entity, so that ungrouping in one canvas doesn't delete the entity
+                    #   group in another.
+                    entityJSON = self.tabbedPane.entityDB.getEntity(groupEntityToSend)
+                    # Dereference the list, so we don't have issues w/ the original Group Node.
+                    newEntity = self.tabbedPane.entityDB.addEntity(
+                        {'Group Name': 'Entity Group', 'Child UIDs': list(entityJSON['Child UIDs']),
+                         'Entity Type': 'EntityGroup'})
+                    newUID = newEntity['uid']
 
-            self.tabbedPane.canvasTabs[otherCanvasName].scene().rearrangeGraph()
+                    # Ensure that no duplicate nodes exist.
+                    newGroupNode = otherCanvas.addNodeProgrammatic(newUID, newEntity['Child UIDs'])
+                    if newGroupNode is None:
+                        self.tabbedPane.mainWindow.MESSAGEHANDLER.info("Cannot send group node to other canvas: The "
+                                                                       "nodes it contains already exist there! Make "
+                                                                       "sure that the nodes in the group node you're "
+                                                                       "trying to send don't exist inside other groups "
+                                                                       "at the destination canvas.", popUp=True)
+                        self.tabbedPane.entityDB.removeEntity(newUID)
+
+                otherCanvas.rearrangeGraph()
+            else:
+                self.tabbedPane.mainWindow.MESSAGEHANDLER.info("Please select a valid Canvas name, "
+                                                               "or create a new Canvas.", popUp=True)
 
     def takePictureOfView(self, justViewport: bool = True, transparentBackground: bool = False) -> QtGui.QImage:
         # Need to set size and format of pic before using it.
@@ -976,13 +1012,13 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         self.nodesDict[item.uid] = item
         self.addItem(item)
         item.setPos(QtCore.QPointF(x, y))
-        self.parent().mainWindow.MESSAGEHANDLER.info('Added node: ' + str(item.uid))
+        self.parent().mainWindow.MESSAGEHANDLER.info('Added node: ' + str(item.uid) + ' | ' + item.labelItem.text())
 
     def addLinkToScene(self, link: Entity.BaseConnector) -> None:
         self.linksDict[link.startItem().uid + link.endItem().uid] = link
         self.addItem(link)
         self.parent().mainWindow.MESSAGEHANDLER.info('Added link: (' + link.startItem().uid + ", " +
-                                                     link.endItem().uid + ')')
+                                                     link.endItem().uid + ') | ' + link.labelItem.text())
 
     def appendSelectedItemsToGroupToggle(self) -> None:
         if self.linking:
@@ -1135,6 +1171,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             self.addEntityLinkCreatorHelper(self.nodesDict[entity])
 
         progress.setValue(steps)
+        self.parent().mainWindow.MESSAGEHANDLER.info('Loaded canvas ' + canvasName)
 
     def updatePositionInDB(self, uid, x, y) -> None:
         """
@@ -1151,34 +1188,37 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         newNodePos = (x, y)
         picture = entity.get('Icon')
 
-        if uid in self.sceneGraph.nodes:
-            del self.sceneGraph.nodes[uid]['groupID']
-        else:
-            self.sceneGraph.add_node(uid)
-        self.scenePos[uid] = newNodePos
         try:
             nodePrimaryAttribute = entity.get(list(entity)[1])
         except IndexError:
             nodePrimaryAttribute = ''
 
+        newNode = None
         if entity.get('Entity Type') == "EntityGroup":
-            newNode = Entity.GroupNode(picture, uid)
-            self.addNodeToScene(newNode, x, y)
-            groupItems = [uid for uid in entity['Child UIDs'] if uid not in self.nodesDict]
+            groupItems = [uid for uid in entity['Child UIDs'] if uid not in self.sceneGraph.nodes]
+            if len(groupItems) > 0:
+                newNode = Entity.GroupNode(picture, uid)
+                self.addNodeToScene(newNode, x, y)
 
-            newGroupList = newNode.listWidget
-            newGroupListGraphic = self.addWidget(newGroupList)
-            newGroupListGraphic.hide()
-            newNode.formGroup(groupItems, newGroupListGraphic)
-            for item in groupItems:
-                self.sceneGraph.add_node(item, groupID=newNode.uid)
+                newGroupList = newNode.listWidget
+                newGroupListGraphic = self.addWidget(newGroupList)
+                newGroupListGraphic.hide()
+                newNode.formGroup(groupItems, newGroupListGraphic)
+                for item in groupItems:
+                    self.sceneGraph.add_node(item, groupID=newNode.uid)
         else:
             if not fromServer:
                 self.parent().mainWindow.sendLocalCanvasUpdateToServer(self.getSelfName(), uid)
             newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
             self.addNodeToScene(newNode, x, y)
 
-        self.addEntityLinkCreatorHelper(newNode)
+        if newNode is not None:
+            if uid in self.sceneGraph.nodes:
+                del self.sceneGraph.nodes[uid]['groupID']
+            else:
+                self.sceneGraph.add_node(uid)
+            self.scenePos[uid] = newNodePos
+            self.addEntityLinkCreatorHelper(newNode)
 
         # Need to return entity Json to show the property editor if new
         #   entity was added.
@@ -1191,12 +1231,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
 
         picture = entity.get('Icon')
 
-        if uid in self.sceneGraph.nodes:
-            del self.sceneGraph.nodes[uid]['groupID']
-        else:
-            self.sceneGraph.add_node(uid)
-        self.sceneGraph.add_node(uid)
-
+        newNode = None
         if groupItems is None:
             # Do not sync group entities.
             if not fromServer:
@@ -1208,18 +1243,24 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             newNode = Entity.BaseNode(picture, uid, nodePrimaryAttribute)
             self.addNodeToScene(newNode)
         else:
-            groupItems = [uid for uid in groupItems if uid not in self.nodesDict]
-            newNode = Entity.GroupNode(picture, uid)
-            self.addNodeToScene(newNode)
+            groupItems = [uid for uid in groupItems if uid not in self.sceneGraph.nodes]
+            if len(groupItems) > 0:
+                newNode = Entity.GroupNode(picture, uid)
+                self.addNodeToScene(newNode)
 
-            newGroupList = newNode.listWidget
-            newGroupListGraphic = self.addWidget(newGroupList)
-            newGroupListGraphic.hide()
-            newNode.formGroup(groupItems, newGroupListGraphic)
-            for item in groupItems:
-                self.sceneGraph.add_node(item, groupID=newNode.uid)
-
-        self.addEntityLinkCreatorHelper(newNode)
+                newGroupList = newNode.listWidget
+                newGroupListGraphic = self.addWidget(newGroupList)
+                newGroupListGraphic.hide()
+                newNode.formGroup(groupItems, newGroupListGraphic)
+                for item in groupItems:
+                    self.sceneGraph.add_node(item, groupID=newNode.uid)
+        if newNode is not None:
+            if uid in self.sceneGraph.nodes:
+                del self.sceneGraph.nodes[uid]['groupID']
+            else:
+                self.sceneGraph.add_node(uid)
+            self.sceneGraph.add_node(uid)
+            self.addEntityLinkCreatorHelper(newNode)
 
         return newNode
 
@@ -1299,7 +1340,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         linkStringUID = origin.uid + destination.uid
         if linkStringUID in self.linksDict:
             linkToEdit = self.linksDict[linkStringUID]
-            if linkToEdit.labelItem.text() != name:
+            if linkToEdit.labelItem.text() != name and linkUID not in linkToEdit.uid:
                 linkToEdit.updateLabel('')
             linkToEdit.uid.add(linkUID)
         else:
@@ -1322,7 +1363,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         linkStringUID = parentItem.uid + childItem.uid
         if linkStringUID in self.linksDict:
             linkToEdit = self.linksDict[linkStringUID]
-            if linkToEdit.labelItem.text() != name:
+            if linkToEdit.labelItem.text() != name and uid not in linkToEdit.uid:
                 linkToEdit.updateLabel('')
             linkToEdit.uid.add(uid)
         else:
@@ -1411,6 +1452,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
 
             item.iconItem.setPos(item.pos())
             item.addToGroup(item.iconItem)
+            primaryField = pEditor.objectJson[list(pEditor.objectJson)[1]]
+            self.parent().messageHandler.info('Edited node: ' + pEditor.objectJson['uid'] + ' | ' + primaryField)
 
     def editLinkProperties(self, linkUID: tuple) -> None:
         """
@@ -1427,6 +1470,8 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         if pEditor.exec_():
             # Adding link with the same UID just overwrites properties.
             self.parent().entityDB.addLink(pEditor.objectJson)
+            self.parent().messageHandler.info('Edited link: ' + str(pEditor.objectJson['uid']) + ' | ' +
+                                              pEditor.objectJson['Resolution'])
 
     # Because the entities on each canvas are stored in dicts, and dicts are ordered, group nodes will always
     # come after the nodes they contain.
@@ -1445,14 +1490,13 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             uid = newEntity['uid']
             [self.removeNode(item) for item in items]
             groupNode = self.addNodeProgrammatic(uid, itemUIDs)
-            # [self.sceneGraph.add_node(itemUID, groupID=uid) for itemUID in itemUIDs]
-            self.rearrangeGraph()
+            if groupNode is not None:
+                self.rearrangeGraph()
             return groupNode
         else:
             self.parent().mainWindow.setStatus('Please select more than one node to create a Group node.')
 
     def ungroupSelectedItems(self) -> None:
-        # groupNodes = [item for item in self.selectedItems() if isinstance(item, Entity.GroupNode)]
         groups = [item for item in self.selectedItems() if isinstance(item, Entity.GroupNode)]
         for group in groups:
             newNodesUIDs = group.groupedNodesUid
@@ -1523,14 +1567,11 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             self.removeItem(edgeItem)
         del edgeItem
 
-    def syncCanvas(self, newCanvas: nx.DiGraph) -> None:
+    def syncCanvas(self, canvas_nodes: dict, canvas_edges: dict) -> None:
         """
         Merge canvas updates from remote client into this canvas.
-
-        :param newCanvas:
-        :return:
         """
-        newNodes = [node for node in newCanvas.nodes() if node not in self.nodesDict and
+        newNodes = [node for node in canvas_nodes if node not in self.nodesDict and
                     self.parent().mainWindow.LENTDB.getEntity(node)['Entity Type'] != 'EntityGroup']
         # newGroupNodes = [y for y in newNodes
         #                 if self.parent().mainWindow.LENTDB.getEntity(y)['Entity Type'] == 'EntityGroup']
@@ -1544,10 +1585,10 @@ class CanvasScene(QtWidgets.QGraphicsScene):
 
         # Edges technically only added if the related nodes are already created,
         #   otherwise addNodeProgrammatic should add them automatically.
-        edges = [edge for edge in newCanvas.edges() if (edge not in self.linksDict) and
+        edges = [edge for edge in canvas_edges if (edge[0] + edge[1] not in self.linksDict) and
                  (edge[0] in self.nodesDict and edge[1] in self.nodesDict)]
         for edge in edges:
-            self.addLinkProgrammatic(edge, newCanvas.edges.get(edge)['Resolution'], fromServer=True)
+            self.addLinkProgrammatic(edge, canvas_edges.get(edge)['Resolution'], fromServer=True)
         self.rearrangeGraph()
 
 
@@ -1687,6 +1728,7 @@ class SendToOtherTabCanvasSelector(QtWidgets.QDialog):
 
     def __init__(self, canvasNames: list):
         super(SendToOtherTabCanvasSelector, self).__init__()
+        self.setStyleSheet(Stylesheets.MAIN_WINDOW_STYLESHEET)
         self.setModal(True)
         self.setWindowTitle('Move Selected Entities to New Canvas')
 

Core/Interface/DockBarThree.py

@@ -4,7 +4,9 @@ from PySide6 import QtWidgets, QtCore, QtCharts, QtGui
 from Core.Interface import Stylesheets
 from datetime import datetime
 from getpass import getuser
+from time import sleep
 import networkx as nx
+import queue
 
 
 class DockBarThree(QtWidgets.QDockWidget):
@@ -16,12 +18,17 @@ class DockBarThree(QtWidgets.QDockWidget):
         # self.setStyleSheet(Stylesheets.MAIN_WINDOW_STYLESHEET)
         childWidget = QtWidgets.QWidget()
         childWidget.setLayout(QtWidgets.QVBoxLayout())
+        childWidget.setContentsMargins(0, 0, 0, 0)
         self.setWidget(childWidget)
 
         childWidget2 = QtWidgets.QWidget()
+        childWidget2.setContentsMargins(0, 0, 0, 0)
         childWidget2.setLayout(QtWidgets.QHBoxLayout())
         childWidget.layout().addWidget(childWidget2)
-        childWidget2.layout().addWidget(self.timeWidget)
+        childWidget2.layout().addWidget(self.tabPane)
+        self.tabPane.setContentsMargins(0, 0, 0, 0)
+        self.tabPane.addTab(self.logViewer, 'Program Log')
+        self.tabPane.addTab(self.timeWidget, 'Timeline')
         childWidget2.layout().addWidget(self.chatBox)
         self.serverStatus.setStyleSheet(Stylesheets.DOCK_BAR_LABEL)
         childWidget.layout().addWidget(self.serverStatus)
@@ -38,12 +45,25 @@ class DockBarThree(QtWidgets.QDockWidget):
         self.setMaximumHeight(275)
         self.setMinimumHeight(275)
 
+        self.tabPane = QtWidgets.QTabWidget()
+
         self.serverStatus = ServerStatusBox(self)
         self.chatBox = ChatBox(self, self.parent())
         self.timeWidget = TimeWidget(self, self.parent())
+        self.logViewer = QtWidgets.QPlainTextEdit()
+
+        # Because we're not going to stop the thread before closing, an error will be thrown by Qt.
+        # That error can be safely ignored.
+        self.logViewerUpdateThread = LoggingUpdateThread(mainWindow.MESSAGEHANDLER)
+        self.logViewerUpdateThread.loggingSignal.connect(self.updateLogs)
+        self.logViewerUpdateThread.start()
+        self.logViewer.setReadOnly(True)
 
         self.initialiseLayout()
 
+    def updateLogs(self, newLogMessage: str):
+        self.logViewer.appendPlainText(newLogMessage)
+
 
 class TimeWidget(QtWidgets.QWidget):
 
@@ -236,7 +256,7 @@ class TimeWidget(QtWidgets.QWidget):
 
         yAxis = QtCharts.QValueAxis()
         yAxis.applyNiceNumbers()
-        yAxis.setTickCount(min(maxEntityNum + 1, 5))
+        yAxis.setTickCount(min(maxEntityNum + 1, 4))
 
         xAxis = QtCharts.QBarCategoryAxis()
         xAxis.append(xAxisValues)
@@ -502,3 +522,25 @@ class ChatBox(QtWidgets.QWidget):
         self.mainWindow.sendChatMessage(self.chatName + self.textSendBox.text())
         self.receiveMessage(self.chatName + self.textSendBox.text())
         self.textSendBox.setText("")
+
+
+class LoggingUpdateThread(QtCore.QThread):
+    loggingSignal = QtCore.Signal(str)
+    endLogging = False
+
+    def __init__(self, messageHandler):
+        super().__init__()
+        self.messageHandler = messageHandler
+
+    def run(self):
+        while True:
+            if self.endLogging:
+                break
+            if not self.messageHandler.logQueue.empty():
+                try:
+                    logMsg = self.messageHandler.logQueue.get().getMessage()
+                    self.loggingSignal.emit(logMsg)
+                except queue.Empty:
+                    pass
+            else:
+                self.msleep(100)

Core/Interface/MenuBar.py

@@ -1,28 +1,35 @@
 #!/usr/bin/env python3
 
+
+import hashlib
 import re
 import json
 import platform
+import sqlite3
+import tempfile
+import shutil
 import os
+import time
 
 import magic
 import lz4.block
 import pandas as pd
 import webbrowser
+from typing import Union
 from urllib import parse
 from pathlib import Path
+from datetime import datetime
 
-from playwright.sync_api import sync_playwright, Error
+from playwright.sync_api import sync_playwright, Error, TimeoutError
 
 from PySide6 import QtWidgets, QtGui, QtCore
-
 from Core.Interface import Stylesheets
 from Core.Interface.Entity import BaseNode
 
 
 class MenuBar(QtWidgets.QMenuBar):
 
-    def __init__(self, parent=None):
+    def __init__(self, parent):
         super().__init__(parent=parent)
 
         fileMenu = self.addMenu("File")
@@ -565,17 +572,43 @@ class MenuBar(QtWidgets.QMenuBar):
                                         if not url.startswith('about:'):
                                             tabsToOpen.add((url, browserEntry['title']))
 
+                            cookiesDatabasePath = tabsFilePath.parent.parent / 'cookies.sqlite'
+                            newCookiesDatabase = tempfile.mkstemp(suffix='.sqlite')
+                            newCookiesDatabasePath = Path(newCookiesDatabase[1])
+
+                            # Try to copy the database a few times, so we can access it
+                            #   with sqlite (original is locked)
+                            copiedFile = False
+                            for _ in range(5):
+                                shutil.copyfile(cookiesDatabasePath, newCookiesDatabasePath)
+                                originalDigest = self.cookieFileHashHelper(cookiesDatabasePath)
+                                newDigest = self.cookieFileHashHelper(newCookiesDatabasePath)
+                                if originalDigest == newDigest:
+                                    copiedFile = True
+                                    break
+                                else:
+                                    time.sleep(0.2)
+
                             browserCookies = []
-                            for cookie in tabsJson['cookies']:
-                                newCookie = {'name': cookie['name'], 'value': cookie['value'],
-                                             'domain': cookie['domain'], 'path': cookie['path'],
-                                             'httpOnly': cookie.get('httponly', False),
-                                             'secure': cookie.get('secure', False)}
-                                if cookie.get('expiry', None) is not None:
-                                    newCookie['expires'] = float(cookie['expiry'])
-                                if cookie.get('sameSite', None) is not None:
-                                    newCookie['sameSite'] = cookie['sameSite']
-                                browserCookies.append(newCookie)
+                            if not copiedFile:
+                                self.parent().MESSAGEHANDLER.warning('Could not access Firefox cookies.', popUp=True)
+                            else:
+                                cookiesDB = sqlite3.connect(newCookiesDatabasePath)
+                                for cookie in cookiesDB.execute('SELECT name,value,host,path,expiry,isSecure,'
+                                                                'isHttpOnly,sameSite FROM moz_cookies'):
+                                    newCookie = {'name': cookie[0], 'value': cookie[1],
+                                                 'domain': cookie[2], 'path': cookie[3],
+                                                 'expires': cookie[4], 'secure': bool(cookie[5]),
+                                                 'httpOnly': bool(cookie[6])}
+                                    if cookie[7] == 0:
+                                        newCookie['sameSite'] = "None"
+                                    elif cookie[7] == 1:
+                                        newCookie['sameSite'] = "Lax"
+                                    elif cookie[7] == 2:
+                                        newCookie['sameSite'] = "Strict"
+                                    browserCookies.append(newCookie)
+                                cookiesDB.close()
+                            newCookiesDatabasePath.unlink(missing_ok=True)
                             context.add_cookies(browserCookies)
                             page = context.new_page()
 
@@ -618,7 +651,11 @@ class MenuBar(QtWidgets.QMenuBar):
                                                            'Entity Type': 'Website'}])
 
                                     if importDialog.importScreenshotsCheckbox.isChecked():
-                                        page.goto(actualURL)
+                                        # If we time out, take a screenshot of the page as-is.
+                                        try:
+                                            page.goto(actualURL)
+                                        except TimeoutError:
+                                            pass
 
                                         urlSaveDir = projectFilesDir / urlTitle
 
@@ -629,20 +666,22 @@ class MenuBar(QtWidgets.QMenuBar):
                                             urlSaveDir = None
 
                                         if urlSaveDir is not None:
-                                            screenshotSavePath = str(urlSaveDir / (actualURL.replace('/', '+') +
-                                                                                   ' screenshot.png'))
+                                            timeNow = str(datetime.now().timestamp() * 1000000).split('.')[0]
+                                            screenshotSavePath = str(urlSaveDir / (
+                                                    actualURL.replace('/', '+') + ' ' + timeNow + ' screenshot.png'))
                                             page.screenshot(path=screenshotSavePath, full_page=True)
 
                                             returnResults.append(
-                                                [{'Image Name': urlTitle + ' Website Screenshot',
+                                                [{'Image Name': decodedPath + ' Screenshot ' + timeNow,
                                                   'File Path': screenshotSavePath,
                                                   'Entity Type': 'Image'},
-                                                 {'Resolution': 'Screenshot of Tab', 'Notes': ''}])
+                                                 {len(returnResults) - 1: {'Resolution': 'Screenshot of Tab',
+                                                                           'Notes': ''}}])
 
                         browser.close()
-                    except Error:
-                        self.parent().MESSAGEHANDLER.warning('Firefox executable is not installed. Cannot import '
-                                                             'tabs from Firefox.', popUp=True)
+                    except Error as e:
+                        self.parent().MESSAGEHANDLER.warning('Cannot import tabs from Firefox: ' + str(repr(e)),
+                                                             popUp=True)
 
                 progress.setValue(2)
                 if importDialog.chromeChoice.isChecked() and not progress.wasCanceled():
@@ -667,7 +706,7 @@ class MenuBar(QtWidgets.QMenuBar):
                                 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                                            "(KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"
                             )
-                            sessionFilePath = Path.home() / 'AppData' / 'Local' / 'Google' / 'Chrome' / 'User Data' /\
+                            sessionFilePath = Path.home() / 'AppData' / 'Local' / 'Google' / 'Chrome' / 'User Data' / \
                                               'Default'
 
                         lastSessionOpenTabs = set()
@@ -767,8 +806,11 @@ class MenuBar(QtWidgets.QMenuBar):
                                                            'Entity Type': 'Website'}])
 
                                     if importDialog.importScreenshotsCheckbox.isChecked():
-                                        page.goto(tabURL)
-
+                                        # If we time out, take a screenshot of the page as-is.
+                                        try:
+                                            page.goto(tabURL)
+                                        except TimeoutError:
+                                            pass
                                         urlSaveDir = projectFilesDir / urlTitle
 
                                         try:
@@ -778,15 +820,17 @@ class MenuBar(QtWidgets.QMenuBar):
                                             urlSaveDir = None
 
                                         if urlSaveDir is not None:
+                                            timeNow = str(datetime.now().timestamp() * 1000000).split('.')[0]
                                             screenshotSavePath = str(urlSaveDir / (
-                                                        tabURL.replace('/', '+') + ' screenshot.png'))
+                                                    tabURL.replace('/', '+') + ' ' + timeNow + ' screenshot.png'))
                                             page.screenshot(path=screenshotSavePath, full_page=True)
 
                                             returnResults.append(
-                                                [{'Image Name': urlTitle + ' Website Screenshot',
+                                                [{'Image Name': decodedPath + ' Screenshot ' + timeNow,
                                                   'File Path': screenshotSavePath,
                                                   'Entity Type': 'Image'},
-                                                 {'Resolution': 'Screenshot of Tab', 'Notes': ''}])
+                                                 {len(returnResults) - 1: {'Resolution': 'Screenshot of Tab',
+                                                                           'Notes': ''}}])
 
                         browser.close()
                     except Error:
@@ -794,38 +838,110 @@ class MenuBar(QtWidgets.QMenuBar):
                                                              'import tabs from Chrome / Chromium.', popUp=True)
 
             progress.setValue(3)
-            newNodeUIDs = []
-            newLinks = []
             if progress.wasCanceled():
                 progress.setValue(4)
                 self.parent().setStatus('Cancelled importing entities from Browser.')
                 return
-
-            for newNode in returnResults:
-                newNodeJson = self.parent().LENTDB.addEntity(newNode[0])
-                if newNodeJson is not None:
-                    newNodeUIDs.append(newNodeJson['uid'])
-                else:
-                    newNodeUIDs.append(None)
-                if len(newNode) > 1:
-                    parentUID = newNodeUIDs[-2]
-                    if parentUID is not None:
-                        newLink = (parentUID, newNodeJson['uid'], newNode[1]['Resolution'], newNode[1]['Notes'])
-                        newLinks.append(newLink)
-
-            self.parent().centralWidget().tabbedPane.linkAddHelper(newLinks)
-            self.parent().setStatus('Imported entities from Browser.')
-
-            if importDialog.importToCanvasCheckbox.isChecked():
-                sceneToAddTo = self.parent().centralWidget().tabbedPane.getSceneByName(
-                    importDialog.importToCanvasDropdown.currentText())
-                for newNodeUID in newNodeUIDs:
-                    if newNodeUID is not None:
-                        sceneToAddTo.addNodeProgrammatic(newNodeUID)
-                sceneToAddTo.rearrangeGraph()
-                self.parent().setStatus('Imported entities from Browser into Canvas.')
+            if returnResults:
+                self.importBrowserTabsFindings(returnResults, importDialog.importToCanvasCheckbox.isChecked(),
+                                               importDialog.importToCanvasDropdown.currentText())
             progress.setValue(4)
 
+    def cookieFileHashHelper(self, filePath):
+        cookieHash = hashlib.md5()  # nosec
+        with open(filePath, 'rb') as cookieFile:
+            for chunk in iter(lambda: cookieFile.read(4096), b""):
+                cookieHash.update(chunk)
+        return cookieHash.digest()
+
+    def importBrowserTabsFindings(self, resolution_result: list, importToCanvas: Union[bool, None] = None,
+                                  canvasToImportTo: Union[str, None] = None) -> None:
+        # See the function 'facilitateResolution' in CentralPane for guidance.
+
+        # Get all the entities, then split it into several lists, to make searching & iterating through them faster.
+        allEntities = [(entity['uid'], (entity[list(entity)[1]], entity['Entity Type']))
+                       for entity in self.parent().LENTDB.getAllEntities()]
+        if allEntities:
+            allEntityUIDs, allEntityPrimaryFieldsAndTypes = map(list, zip(*allEntities))
+        else:
+            allEntityUIDs = []
+            allEntityPrimaryFieldsAndTypes = []
+        allLinks = [linkUID['uid'] for linkUID in self.parent().LENTDB.getAllLinks()]
+        links = []
+        newNodeUIDs = []
+        for resultList in resolution_result:
+            newNodeJSON = resultList[0]
+            newNodeEntityType = newNodeJSON['Entity Type']
+            # Cannot assume proper order of dicts sent over the net.
+            newNodePrimaryFieldKey = self.parent().RESOURCEHANDLER.getPrimaryFieldForEntityType(newNodeEntityType)
+            newNodePrimaryField = newNodeJSON[newNodePrimaryFieldKey]
+
+            try:
+                # Attempt to get the index of an existing entity that shares primary field and type with the new
+                #   entity. Those two entities are considered to be referring to the same thing.
+                newNodeExistsIndex = allEntityPrimaryFieldsAndTypes.index((newNodePrimaryField, newNodeEntityType))
+                # If entity already exists, update the fields and re-add
+                newNodeExistingUID = allEntityUIDs[newNodeExistsIndex]
+                existingEntityJSON = self.parent().LENTDB.getEntity(newNodeExistingUID)
+                # Remove primary field and entity type, since those are duplicates. Primary field is the first element.
+                del newNodeJSON['Entity Type']
+                del newNodeJSON[newNodePrimaryFieldKey]
+                try:
+                    notesField = newNodeJSON.pop('Notes')
+                    existingEntityJSON['Notes'] += '\n' + notesField
+                except KeyError:
+                    # If no new field was actually added to the entity, don't re-add to the database
+                    if len(newNodeJSON) == 0:
+                        newNodeUIDs.append(newNodeExistingUID)
+                        continue
+                # Update old values to new ones, and add new ones where applicable.
+                existingEntityJSON.update(dict((newNodeKey, newNodeJSON[newNodeKey]) for newNodeKey in newNodeJSON))
+                self.parent().LENTDB.addEntity(existingEntityJSON, fromServer=True, updateTimeline=False)
+                newNodeUIDs.append(newNodeExistingUID)
+            except ValueError:
+                # If there is no index for which the primary field and entity type of the new node match one of the
+                #   existing ones, the node must indeed be new. We add it here.
+                entityJson = self.parent().LENTDB.addEntity(newNodeJSON, fromServer=True, updateTimeline=False)
+                newNodeUIDs.append(entityJson['uid'])
+                # Ensure that different entities involved in the resolution can't independently
+                #   create the same new entities.
+                allEntityUIDs.append(entityJson['uid'])
+                allEntityPrimaryFieldsAndTypes.append((newNodePrimaryField, newNodeEntityType))
+
+        for resultListIndex in range(len(resolution_result)):
+            if len(resolution_result[resultListIndex]) > 1:
+                outputEntityUID = newNodeUIDs[resultListIndex]
+                parentsDict = resolution_result[resultListIndex][1]
+                for parentID in parentsDict:
+                    parentUID = parentID
+                    if isinstance(parentUID, int):
+                        parentUID = newNodeUIDs[parentUID]
+                    resolutionName = parentsDict[parentID]['Resolution']
+                    newLinkUID = (parentUID, outputEntityUID)
+                    # Avoid creating more links between the same two entities.
+                    if newLinkUID in allLinks:
+                        linkJson = self.parent().LENTDB.getLinkIfExists(newLinkUID)
+                        if resolutionName not in linkJson['Notes']:
+                            linkJson['Notes'] += '\nConnection also produced by Resolution: ' + resolutionName
+                            self.parent().LENTDB.addLink(linkJson, fromServer=True)
+                    else:
+                        self.parent().LENTDB.addLink({'uid': newLinkUID, 'Resolution': resolutionName,
+                                                      'Notes': parentsDict[parentID]['Notes']}, fromServer=True)
+                        links.append((parentUID, outputEntityUID, resolutionName))
+                        allLinks.append(newLinkUID)
+
+        self.parent().syncDatabase()
+        if importToCanvas:
+            sceneToAddTo = self.parent().centralWidget().tabbedPane.getSceneByName(canvasToImportTo)
+            for newNodeUID in newNodeUIDs:
+                if newNodeUID is not None and newNodeUID not in sceneToAddTo.sceneGraph.nodes:
+                    sceneToAddTo.addNodeProgrammatic(newNodeUID)
+            sceneToAddTo.rearrangeGraph()
+        self.parent().centralWidget().tabbedPane.addLinksToTabs(links, "Browser Import")
+        self.parent().LENTDB.resetTimeline()
+        self.parent().saveProject()
+        self.parent().MESSAGEHANDLER.info('Imported tabs from browser successfully.')
+
 
 class BrowserImportDialog(QtWidgets.QDialog):
 
@@ -1042,6 +1158,7 @@ class ViewAndStopResolutionsDialogOption(QtWidgets.QPushButton):
 
     def __init__(self, resolutionThread, fromServer, mainWindowObject):
         super(ViewAndStopResolutionsDialogOption, self).__init__()
+        self.setStyleSheet(Stylesheets.MAIN_WINDOW_STYLESHEET)
         self.mainWindowObject = mainWindowObject
         self.resolutionThread = resolutionThread
         self.fromServer = fromServer

Core/MessageHandler.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 
 import logging
+from logging import handlers
 
+from multiprocessing import Queue
 from pathlib import Path
 from PySide6 import QtWidgets
 
@@ -82,16 +84,21 @@ class MessageHandler:
 
     def __init__(self, parentObject):
         self.mainWindow = parentObject
+        self.logQueue = Queue()
         self.logFileHandler = logging.FileHandler(
             self.mainWindow.SETTINGS.value("Logging/Logfile", str(Path.home() / 'LinkScope_logfile.log')), 'a')
+        self.logQueueHandler = handlers.QueueHandler(self.logQueue)
         self.logFormatter = logging.Formatter(
             '{' + self.mainWindow.SETTINGS.value("Project/Name", "Untitled") + '} [%(asctime)s] - %(levelname)s: %('
                                                                                'message)s')
+        self.logFormatterQueue = logging.Formatter('[%(asctime)s] - %(levelname)s: %(message)s')
         self.logFileHandler.setFormatter(self.logFormatter)
+        self.logQueueHandler.setFormatter(self.logFormatterQueue)
         rootLogger = logging.getLogger()
         for handler in rootLogger.handlers[:]:
             if isinstance(handler, logging.FileHandler):
                 rootLogger.removeHandler(handler)
         rootLogger.addHandler(self.logFileHandler)
+        rootLogger.addHandler(self.logQueueHandler)
 
         self.setSeverityLevel(self.mainWindow.SETTINGS.value("Logging/Severity", logging.INFO))

Core/ResolutionManager.py

@@ -115,7 +115,8 @@ class ResolutionManager:
                     if category == "Server Resolutions":
                         self.mainWindow.executeRemoteResolution(resolutionName, resolutionEntitiesInput, parameters,
                                                                 resolutionUID)
-                        return None
+                        # Returning a bool so we know that the resolution is running on the server.
+                        return True
 
                     resolutionClass = self.resolutions[category][resolution]['resolution']()
                     result = resolutionClass.resolution(resolutionEntitiesInput, parameters)

Core/Resolutions/Core/FileExtractor.py

@@ -109,7 +109,7 @@ class FileExtractor:
                                                          'Notes': ''}}])
 
                             docProperName = link.split('/')[-1]
-                            docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName
+                            docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
                             docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
                             try:
@@ -151,7 +151,7 @@ class FileExtractor:
                                               {uid: {'Resolution': 'File URL',
                                                      'Notes': ''}}])
                         docProperName = link.split('/')[-1]
-                        docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName
+                        docFileName = hexlify(md5(link.encode()).digest()).decode() + ' | ' + docProperName  # nosec
                         docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
                         try:

Core/Resolutions/Core/FileHasher.py

@@ -25,11 +25,11 @@ class FileHasher:
             block_size = 65536  # The size of each read from the file
             for hashing_algorithm in hashing_algorithms:
                 if hashing_algorithm == "SHA256":
-                    file_hash = hashlib.sha256()
+                    file_hash = hashlib.sha256()  # nosec
                 elif hashing_algorithm == "SHA1":
-                    file_hash = hashlib.sha1()
+                    file_hash = hashlib.sha1()  # nosec
                 else:
-                    file_hash = hashlib.md5()
+                    file_hash = hashlib.md5()  # nosec
                 with open(file_path, 'rb') as f:
                     fb = f.read(block_size)
                     while len(fb) > 0:

Core/ResourceHandler.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
 
+import networkx as nx
 from defusedxml.ElementTree import parse
 from datetime import datetime
 from os import listdir
 from pathlib import Path
 from uuid import uuid4
+from ast import literal_eval
+from base64 import b64decode
 
 from PySide6.QtGui import QIcon
 from PySide6.QtCore import QByteArray
@@ -243,3 +246,26 @@ class ResourceHandler:
     def getLinkArrowPicture(self):
         picture = Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Resources" / "Icons" / "Right-Arrow.svg"
         return QIcon(str(picture)).pixmap(40, 40)
+
+    def deconstructGraph(self, graph: nx.DiGraph) -> tuple:
+        nodes = {}
+        for nodeKey in graph.nodes:
+            # Dereference the original dict so we don't actually convert its icon to data.
+            nodes[nodeKey] = dict(graph.nodes.get(nodeKey))
+            try:
+                nodes[nodeKey]['Icon'] = nodes[nodeKey]['Icon'].toBase64().data()
+            except KeyError:
+                pass
+
+        edges = {edgeKey: graph.edges.get(edgeKey) for edgeKey in graph.edges}
+        return nodes, edges
+
+    def reconstructGraph(self, graphString: str) -> tuple:
+        nodes, edges = literal_eval(graphString)
+        for node in nodes:
+            try:
+                nodes[node]['Icon'] = QByteArray(b64decode(nodes[node]['Icon']))
+            except KeyError:
+                pass
+
+        return nodes, edges

LinkScope.py

@@ -4,6 +4,7 @@
 import sys
 import tempfile
 import threading
+import time
 
 import networkx as nx
 from ast import literal_eval
@@ -18,7 +19,6 @@ from datetime import datetime
 from typing import Union
 from PySide6 import QtWidgets, QtGui, QtCore
 
-import Core.SettingsObject
 from Core import MessageHandler, SettingsObject
 from Core import ResourceHandler
 from Core import ReportGeneration
@@ -42,11 +42,12 @@ class MainWindow(QtWidgets.QMainWindow):
     def centralWidget(self) -> Union[QtWidgets.QWidget, QtWidgets.QWidget, CentralPane.WorkspaceWidget]:
         return super(MainWindow, self).centralWidget()
 
-    def getSettings(self) -> Core.SettingsObject.SettingsObject:
+    def getSettings(self) -> SettingsObject.SettingsObject:
         return self.SETTINGS
 
     # What happens when the software is closed
     def closeEvent(self, event) -> None:
+        self.dockbarThree.logViewerUpdateThread.endLogging = True
         # Save the window settings
         self.SETTINGS.setValue("MainWindow/Geometry", self.saveGeometry())
         self.SETTINGS.setValue("MainWindow/WindowState", self.saveState())
@@ -54,6 +55,10 @@ class MainWindow(QtWidgets.QMainWindow):
             self.FCOM.close()
         self.SETTINGS.setValue("Project/Server/Project", "")
         self.saveProject()
+        # Wait just a little for the logging thread to close.
+        # We don't _have_ to do this, but it stops errors from popping up due to threads being rudely interrupted.
+        while not self.dockbarThree.logViewerUpdateThread.isFinished():
+            time.sleep(0.01)
         super(MainWindow, self).closeEvent(event)
 
     def saveProject(self) -> None:
@@ -62,10 +67,13 @@ class MainWindow(QtWidgets.QMainWindow):
             self.SETTINGS.save()
             self.LENTDB.save()
             self.centralWidget().tabbedPane.save()
-        except Exception:
-            self.MESSAGEHANDLER.error("Could not Save Project.", exc_info=True)
-        self.setStatus("Project Saved.", 3000)
-        self.MESSAGEHANDLER.info('Project Saved')
+            self.setStatus("Project Saved.", 3000)
+            self.MESSAGEHANDLER.info('Project Saved')
+        except Exception as e:
+            errorMessage = "Could not Save Project: " + str(repr(e))
+            self.MESSAGEHANDLER.error(errorMessage, exc_info=True)
+            self.setStatus("Failed Saving Project.", 3000)
+            self.MESSAGEHANDLER.info("Failed Saving Project " + self.SETTINGS.value("Project/Name", 'Untitled'))
 
     def saveAsProject(self) -> None:
         if len(self.resolutions) > 0:
@@ -281,7 +289,9 @@ class MainWindow(QtWidgets.QMainWindow):
 
         self.setWindowTitle("LinkScope - " + self.SETTINGS.get('Project/Name', 'Untitled'))
         self.saveProject()
-        self.setStatus('Project Renamed to: ' + newName)
+        statusMessage = 'Project Renamed to: ' + newName
+        self.setStatus(statusMessage)
+        self.MESSAGEHANDLER.info(statusMessage)
 
     def addCanvas(self) -> None:
         # Create or open canvas
@@ -292,7 +302,8 @@ class MainWindow(QtWidgets.QMainWindow):
         with self.syncedCanvasesLock:
             availableSyncedCanvases = self.syncedCanvases
         newCanvasPopup = CreateOrOpenCanvas(self, connected, availableSyncedCanvases)
-        newCanvasPopup.exec()
+        if newCanvasPopup.exec():
+            self.MESSAGEHANDLER.info("New Canvas added: " + newCanvasPopup.canvasName)
 
     def toggleWorldDoc(self) -> None:
         if self.centralWidget() is not None:
@@ -321,6 +332,7 @@ class MainWindow(QtWidgets.QMainWindow):
             if itemUID in scene.nodesDict:
                 scene.removeNode(scene.nodesDict[itemUID])
         self.LENTDB.removeEntity(itemUID)
+        self.MESSAGEHANDLER.info("Deleted node: " + itemUID)
 
     def deleteSpecificLink(self, linkUIDs: set) -> None:
         """
@@ -336,6 +348,7 @@ class MainWindow(QtWidgets.QMainWindow):
                 scene.removeUIDFromLink(linkUID)
         for linkUID in linkUIDs:
             self.LENTDB.removeLink(linkUID)
+            self.MESSAGEHANDLER.info("Deleted link: " + str(linkUID))
 
     def setGroupAppendMode(self, enable: bool) -> None:
         if self.centralWidget().tabbedPane.getCurrentScene().linking:
@@ -896,6 +909,7 @@ class MainWindow(QtWidgets.QMainWindow):
         resolutionThread = ResolutionExecutorThread(
             resolution, resArgument, resolutionParameterValues, self, resolutionUID)
         resolutionThread.sig.connect(self.resolutionSignalListener)
+        self.MESSAGEHANDLER.info('Running Resolution: ' + resolution)
         resolutionThread.start()
         self.resolutions.append((resolutionThread, category == 'Server Resolutions'))
 
@@ -1090,20 +1104,18 @@ class MainWindow(QtWidgets.QMainWindow):
             self.setStatus('Stopped syncing Canvas: ' + canvasName)
             self.MESSAGEHANDLER.info('Stopped syncing Canvas: ' + canvasName)
 
-    def receiveSyncCanvasListener(self, canvas_name: str, canvas_graph: str) -> None:
+    def receiveSyncCanvasListener(self, canvas_name: str, canvas_nodes: dict, canvas_edges: dict) -> None:
         if canvas_name in self.centralWidget().tabbedPane.canvasTabs:
             canvasToSync = self.centralWidget().tabbedPane.canvasTabs[canvas_name]
             if canvasToSync.synced:
-                canvasToSync.scene().syncCanvas(canvas_graph)
+                canvasToSync.scene().syncCanvas(canvas_nodes, canvas_edges)
                 self.MESSAGEHANDLER.debug('Canvas ' + canvas_name + ' synced.')
 
-    def receiveSyncDatabaseListener(self, database) -> None:
+    def receiveSyncDatabaseListener(self, database_nodes: dict, database_edges: dict) -> None:
         """
         Handles received Database Sync events sent from the server.
-        :param database:
-        :return:
         """
-        self.LENTDB.mergeDatabases(database, fromServer=True)
+        self.LENTDB.mergeDatabases(database_nodes, database_edges, fromServer=True)
         self.MESSAGEHANDLER.debug('Project database synced.')
 
     def sendLocalCanvasUpdateToServer(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]) -> None:
@@ -1119,16 +1131,18 @@ class MainWindow(QtWidgets.QMainWindow):
     def receiveServerCanvasUpdate(self, canvas_name: str, entity_or_link_uid: Union[str, tuple]) -> None:
         scene = self.centralWidget().tabbedPane.getSceneByName(canvas_name)
         if scene is not None:
-            if isinstance(entity_or_link_uid, str):
-                # Add Entity
-                if entity_or_link_uid not in scene.nodesDict:
-                    scene.addNodeProgrammatic(entity_or_link_uid, fromServer=True)
-                    scene.rearrangeGraph()
-            else:
-                # Add Link
-                if entity_or_link_uid not in scene.linksDict:
-                    scene.addLinkProgrammatic(entity_or_link_uid, fromServer=True)
-                    scene.rearrangeGraph()
+            # Check that the argument in the update is not empty.
+            if entity_or_link_uid:
+                if isinstance(entity_or_link_uid, str):
+                    # Add Entity
+                    if entity_or_link_uid not in scene.nodesDict:
+                        scene.addNodeProgrammatic(entity_or_link_uid, fromServer=True)
+                        scene.rearrangeGraph()
+                else:
+                    # Add Link
+                    if entity_or_link_uid not in scene.linksDict:
+                        scene.addLinkProgrammatic(entity_or_link_uid, fromServer=True)
+                        scene.rearrangeGraph()
 
         self.MESSAGEHANDLER.debug('Received update to canvas: ' + canvas_name + ' for entity / link: ' +
                                   str(entity_or_link_uid))
@@ -1147,25 +1161,27 @@ class MainWindow(QtWidgets.QMainWindow):
                                       str(add))
 
     def receiveServerDatabaseUpdate(self, entityJson, add) -> None:
-        uid = entityJson['uid']
-        if add:
-            # Add item
-            if isinstance(uid, str):
-                # Add Entity
-                self.LENTDB.addEntity(entityJson, fromServer=True)
+        # Check that the JSON received is not empty.
+        if entityJson:
+            uid = entityJson['uid']
+            if add:
+                # Add item
+                if isinstance(uid, str):
+                    # Add Entity
+                    self.LENTDB.addEntity(entityJson, fromServer=True)
+                else:
+                    # Add Link
+                    self.centralWidget().tabbedPane.serverLinkAddHelper(entityJson)
             else:
-                # Add Link
-                self.centralWidget().tabbedPane.serverLinkAddHelper(entityJson)
-        else:
-            # Remove item
-            if isinstance(uid, str):
-                # Remove Entity
-                self.centralWidget().tabbedPane.nodeRemoveAllHelper(uid)
-                self.LENTDB.removeEntity(uid, fromServer=True)
-            else:
-                # Remove Link
-                self.centralWidget().tabbedPane.linkRemoveAllHelper(uid)
-                self.LENTDB.removeLink(uid, fromServer=True)
+                # Remove item
+                if isinstance(uid, str):
+                    # Remove Entity
+                    self.centralWidget().tabbedPane.nodeRemoveAllHelper(uid)
+                    self.LENTDB.removeEntity(uid, fromServer=True)
+                else:
+                    # Remove Link
+                    self.centralWidget().tabbedPane.linkRemoveAllHelper(uid)
+                    self.LENTDB.removeLink(uid, fromServer=True)
 
         self.MESSAGEHANDLER.debug('Received database update from server: ' + str(entityJson) + ' - Operation: ' +
                                   str(add))
@@ -1275,15 +1291,17 @@ class MainWindow(QtWidgets.QMainWindow):
                         document_name = selectedDocuments[0].getFileName()
                     else:
                         self.setStatus("Must upload and select document before obtaining summary.")
+                        self.MESSAGEHANDLER.warning("Must upload and select document from 'Files Loaded' section "
+                                                    "before getting summary.", popUp=True)
                 self.FCOM.askServerForFileSummary(project_name, document_name)
             else:
                 self.setStatus("Not Connected to Server.")
-                self.MESSAGEHANDLER.info('Cannot get summary of document ' + document_name +
-                                         ': Not connected to a Server.', popUp=True)
+                self.MESSAGEHANDLER.info('Cannot get summary of documents: Not working on a Server Project.',
+                                         popUp=True)
         else:
             self.setStatus("No currently open Server Project.")
-            self.MESSAGEHANDLER.info('Cannot get summary of document ' + document_name +
-                                     ': Not working on a Server Project.', popUp=True)
+
+            self.MESSAGEHANDLER.info('Cannot get summary of documents: Not connected to a Server.', popUp=True)
 
     def receiveSummaryOfDocument(self, document_name: str, summary: str):
         self.centralWidget().setDocTitleAndContents(document_name, summary)
@@ -1311,17 +1329,17 @@ class MainWindow(QtWidgets.QMainWindow):
                                       int(self.SETTINGS.value("Project/Number of Answers Returned")))
                 self.setStatus("Asked Question")
 
-    def questionAnswerListener(self, response: dict) -> None:
+    def questionAnswerListener(self, response: list) -> None:
         textAns = "No Answer."
-        answerCount = len(response['answers'])
+        answerCount = len(response)
         if answerCount != 0:
             textAns = ""
             for answerIndex in range(answerCount):
-                answer = response['answers'][answerIndex]
-                if answer['answer'] is not None:
+                answer = response[answerIndex]
+                if answer['answer']:
                     textAns += "Answer " + str(answerIndex + 1) + ": " + answer['answer'] + "\n\n"
                     textAns += "Context: ..." + answer['context'] + "...\n\n"
-                    textAns += "Document Used: " + answer['meta']['resourceName'][2:-1]
+                    textAns += "Document Used: " + answer['doc']
                     textAns += "\n\n"
                 else:
                     textAns += "Answer " + str(answerIndex + 1) + ": No Answer\n\n"
@@ -1373,6 +1391,7 @@ class MainWindow(QtWidgets.QMainWindow):
             self.centralWidget().tabbedPane.createHomeTab()
 
         self.setStatus("Ready")
+        self.MESSAGEHANDLER.info('Project opened, ready to work.')
 
     def __init__(self):
         super(MainWindow, self).__init__()
@@ -1897,25 +1916,28 @@ class CreateOrOpenCanvas(QtWidgets.QDialog):
         dialogLayout.addWidget(openServerCanvasNameLabel, 9, 0, 1, 2)
         dialogLayout.addWidget(self.openServerCanvasDropdown, 9, 2, 1, 2)
         dialogLayout.addWidget(openServerCanvasButton, 10, 1, 1, 2)
+        self.canvasName = ""
 
     def confirmCreateCanvas(self):
-        createStatus = self.parent().centralWidget().tabbedPane.addCanvas(self.createCanvasTextbox.text())
+        self.canvasName = self.createCanvasTextbox.text()
+        createStatus = self.parent().centralWidget().tabbedPane.addCanvas(self.canvasName)
         if createStatus:
             self.accept()
         else:
             self.parent().MESSAGEHANDLER.warning("A Canvas with that name already exists!", popUp=True)
 
     def confirmOpenServerCanvas(self):
-        newCanvasName = self.openServerCanvasDropdown.currentText()
-        createStatus = self.parent().centralWidget().tabbedPane.addCanvas(newCanvasName)
+        self.canvasName = self.openServerCanvasDropdown.currentText()
+        createStatus = self.parent().centralWidget().tabbedPane.addCanvas(self.canvasName)
         if createStatus:
-            self.parent().syncCanvasByName(newCanvasName)
+            self.parent().syncCanvasByName(self.canvasName)
             self.accept()
         else:
             self.parent().MESSAGEHANDLER.warning("A Canvas with that name already exists!", popUp=True)
 
     def confirmOpenExistingCanvas(self):
-        self.parent().centralWidget().tabbedPane.showTab(self.openExistingCanvasDropdown.currentText())
+        self.canvasName = self.openExistingCanvasDropdown.currentText()
+        self.parent().centralWidget().tabbedPane.showTab(self.canvasName)
         self.accept()
 
 
@@ -2055,14 +2077,21 @@ class ResolutionExecutorThread(QtCore.QThread):
                                                                       self.resolutionArgument,
                                                                       self.resolutionParameters,
                                                                       self.uid)
+            if ret is None:
+                self.mainWindow.MESSAGEHANDLER.error('Resolution ' + self.resolution + ' failed during run.',
+                                                     popUp=True)
+            elif isinstance(ret, bool):
+                # Resolution is running on the server, we do not have results right now.
+                ret = None
         except Exception as e:
-            self.mainWindow.MESSAGEHANDLER.error('Resolution failed during run: ' + str(e), popUp=False)
+            self.mainWindow.MESSAGEHANDLER.error('Resolution ' + self.resolution + ' failed during run: ' +
+                                                 str(e), popUp=True)
             ret = None
 
         # If the resolution is ran on the server or there is a problem, don't emit signal.
         if ret is not None and self.return_results:
-            self.done = True
             self.sig.emit(self.resolution, ret)
+            self.done = True
 
 
 class ResolutionParametersSelector(QtWidgets.QDialog):

LinuxInstaller.sh

@@ -69,14 +69,15 @@ if [ "$1" == "install" ]; then
   sudo apt install p7zip-full curl libopengl0 graphviz libmagic1 -y
   echo "Downloading latest version of LinkScope client..."
   linuxURL=$(curl -sL https://github.com/AccentuSoft/LinkScope_Client/releases/latest | grep 'Ubuntu-x64.7z' -m 1 | cut -d '"' -f 2 | tr -d ' ')
-  curl -L $linuxURL -o /tmp/LinkScope.7z
+  curl -L https://github.com${linuxURL} -o /tmp/LinkScope.7z
   sudo 7z x /tmp/LinkScope.7z -o/usr/local/sbin/ && rm /tmp/LinkScope.7z
   if [ $? -ne 0 ]; then
     echo "Something went wrong during the download or extraction."
     echo "Please check that /tmp/LinkScope.7z exists, and that it is an archive containing the latest version of the LinkScope Client software."
     exit
   fi
-  sudo echo "$DESKTOP_ENTRY" > /usr/share/applications/LinkScope.desktop
+  sudo echo "$DESKTOP_ENTRY" > /tmp/LinkScope.desktop
+  sudo mv /tmp/LinkScope.desktop /usr/share/applications/LinkScope.desktop
   sudo chmod +x /usr/share/applications/LinkScope.desktop
   read -p "Create a Desktop shortcut? WARNING: This will refresh the desktop! [y/N]" -n 1 -r
   # https://askubuntu.com/a/1014261    -- Making Desktop launchers with .desktop files

Modules/Reddit/Reddit.py

@@ -72,7 +72,7 @@ class Reddit:
                                            'Entity Type': 'Person'},
                                           {index_of_child: {'Resolution': f"https://reddit.com{value['permalink']}",
                                                             'Notes': ''}}])
-                    comment = hashlib.md5(value['body'].encode())
+                    comment = hashlib.md5(value['body'].encode())  # nosec
                     comment = hexlify(comment.digest()).decode()
                     return_result.append([{'Comment': comment,
                                            'Notes': value['body'],

Modules/Shodan/ShodanDomainScan.py

@@ -60,7 +60,7 @@ class ShodanDomainScan:
                                           {uid: {'Resolution': 'Shodan Domain AAAA records', 'Notes': ''}}])
                 elif dns_type == "TXT":
                     # Text records could be massive - do not want them breaking the UI
-                    textPrimaryField = hashlib.md5(value.encode())
+                    textPrimaryField = hashlib.md5(value.encode())  # nosec
                     return_result.append([{'Phrase': primary_field + ' TXT Record: ' +
                                                      hexlify(textPrimaryField.digest()).decode(),
                                            'Entity Type': 'Phrase',

Modules/Username_Crosscheck/Social_Analyzer.py

@@ -40,12 +40,12 @@ class Social_Analyzer:
             try:
                 originalUsernameRegex = re.escape(social_field)
                 originalUsernameRegex2 = re.compile(originalUsernameRegex, re.IGNORECASE)
-                firstResponse = requests.get(original_url, verify=False, timeout=30, allow_redirects=False,
-                                             headers=headers)  # nosec
+                firstResponse = requests.get(original_url, verify=False, timeout=30, allow_redirects=False,  # nosec
+                                             headers=headers)
                 if firstResponse.status_code >= 300:
                     return False
                 else:
-                    modifiedUsername = "".join(random.choices(
+                    modifiedUsername = "".join(random.choices(  # nosec
                         string.ascii_uppercase + string.digits, k=32))
                     usernameRegex = re.compile(social_field, re.IGNORECASE)
                     r = requests.get(original_url, timeout=30, verify=False, headers=headers)  # nosec

Modules/VirusTotal/VirusTotal_Domain.py

@@ -55,7 +55,7 @@ class VirusTotal_Domain:
                                                   {uid: {'Resolution': 'VirusTotal Domain AAAA records', 'Notes': ''}}])
                         elif dns_type == "TXT":
                             # Text records could be massive - do not want them breaking the UI
-                            textPrimaryField = hashlib.md5(value.encode())
+                            textPrimaryField = hashlib.md5(value.encode())  # nosec
                             return_result.append([{'Phrase': primary_field + ' TXT Record: ' +
                                                              hexlify(textPrimaryField.digest()).decode(),
                                                    'Entity Type': 'Phrase',

README.md

@@ -33,9 +33,10 @@ This repository has the code for the Linux version, with the Windows alternative
 
 ### Installing the software
 #### Linux
-An installer is provided for Ubuntu 20.04 and derivatives: https://raw.githubusercontent.com/AccentuSoft/LinkScope_Client/main/LinuxInstaller.sh
+An installer is provided for Ubuntu 20.04 and derivatives.
+To download the installer and install the software, run the following commands in a terminal:
 
-Running the script with the 'install' parameter installs the software, like so:
+`wget https://raw.githubusercontent.com/AccentuSoft/LinkScope_Client/main/LinuxInstaller.sh`
 
 `bash LinuxInstaller.sh install`
 
@@ -53,7 +54,7 @@ One could also clone the repository and run the software as-is.
 Some dependencies need to be installed in order for the software to work properly. After downloading the release correspoding to your platform from the Releases tab, please perform the following steps to install the required dependencies:
 1. Linux
     - `sudo apt update && sudo apt install libopengl0 graphviz libmagic1 -y`
-    - `pip install -r requirements.txt`
+    - `pip install -r requirements.txt --upgrade`
     - `playwright install`
 2. Windows
    - Download and install the graphviz package from https://www.graphviz.org/download/

build.sh

@@ -13,7 +13,7 @@ python3.9 -m pip install --upgrade -r requirements.txt
 
 PLAYWRIGHT_BROWSERS_PATH=0 python3.9 -m playwright install
 
-python3.9 -m PyInstaller --clean --icon="./Icon.ico" --noconsole --noconfirm --onedir --windowed --add-data "./Modules:Modules/" --add-data "./Resources:Resources/" --add-data "./Core:Core/" --collect-all "PySide6" --collect-all "networkx" --collect-all "pydot" --collect-all "msgpack" --hidden-import "_cffi_backend" --collect-all "folium" --collect-all "shodan" --collect-all "vtapi3" --collect-all "docker" --collect-all "exif" --collect-all "dns" --collect-all "pycountry" --collect-all "tldextract" --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" --hidden-import "pandas" --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" --collect-all "Wappalyzer" --collect-all "email_validator" --add-data "./buildEnv/lib/python3.9/site-packages/social-analyzer:social-analyzer/" --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" --hidden-import "logging"  "./LinkScope.py"
+python3.9 -m PyInstaller --clean --icon="./Icon.ico" --noconsole --noconfirm --onedir --windowed --add-data "./Modules:Modules/" --add-data "./Resources:Resources/" --add-data "./Core:Core/" --collect-all "PySide6" --collect-all "networkx" --collect-all "pydot" --collect-all "msgpack" --hidden-import "_cffi_backend" --collect-all "folium" --collect-all "shodan" --collect-all "vtapi3" --collect-all "docker" --collect-all "exif" --collect-all "dns" --collect-all "pycountry" --collect-all "tldextract" --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" --hidden-import "pandas" --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" --collect-all "Wappalyzer" --collect-all "email_validator" --add-data "./buildEnv/lib/python3.9/site-packages/social-analyzer:social-analyzer/" --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" --hidden-import "defusedxml" --hidden-import "cchardet" --hidden-import "ipwhois" --hidden-import "xmltodict" --hidden-import "urllib3" --hidden-import "logging"  "./LinkScope.py"
 
 # Copy web engine resources in final package, so that the map tool works.
 cp buildEnv/lib/python3.9/site-packages/PySide6/Qt/resources/qtwebengine_resources.pak dist/LinkScope

buildWin.bat

@@ -17,7 +17,7 @@ playwright install
 :: Just in case this was not uncommented in the requirements.txt file.
 python -m pip install --upgrade python-magic-bin
 
-python -m PyInstaller --clean --icon="Icon.ico" --noconsole --noconfirm --onedir --windowed --add-data "Modules;Modules" --add-data "Resources;Resources" --add-data "Core;Core" --collect-all "PySide6" --collect-all "networkx" --collect-all "pydot" --collect-all "msgpack" --hidden-import "_cffi_backend" --collect-all "folium" --collect-all "shodan" --collect-all "vtapi3" --collect-all "docker" --collect-all "exif" --collect-all "dns" --collect-all "pycountry" --collect-all "tldextract" --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" --hidden-import "pandas" --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" --collect-all "Wappalyzer" --collect-all "email_validator" --add-data "C:\Users\IEUser\AppData\Roaming\Python\Python39\site-packages\social-analyzer;social-analyzer" --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" --hidden-import "logging" --hidden-import "python-magic-bin" ".\LinkScope.py"
+python -m PyInstaller --clean --icon="Icon.ico" --noconsole --noconfirm --onedir --windowed --add-data "Modules;Modules" --add-data "Resources;Resources" --add-data "Core;Core" --collect-all "PySide6" --collect-all "networkx" --collect-all "pydot" --collect-all "msgpack" --hidden-import "_cffi_backend" --collect-all "folium" --collect-all "shodan" --collect-all "vtapi3" --collect-all "docker" --collect-all "exif" --collect-all "dns" --collect-all "pycountry" --collect-all "tldextract" --collect-all "requests_futures" --collect-all "branca" --collect-all "bs4" --hidden-import "pandas" --collect-all "docx2python" --collect-all "tweepy" --collect-all "PyPDF2" --collect-all "Wappalyzer" --collect-all "email_validator" --add-data "C:\Users\IEUser\AppData\Roaming\Python\Python39\site-packages\social-analyzer;social-analyzer" --hidden-import "PIL" --hidden-import "lz4" --hidden-import "lxml" --hidden-import "jellyfish" --hidden-import "logging" --hidden-import "defusedxml" --hidden-import "xmltodict" --hidden-import "urllib3" --hidden-import "cchardet" --hidden-import "ipwhois" --hidden-import "python-magic-bin" ".\LinkScope.py"
 
 :: Copy web engine resources in final package, so that the map tool works.
 xcopy buildEnv\Lib\site-packages\PySide6\resources\qtwebengine_resources.pak dist\LinkScope /Y

requirements.txt

@@ -16,6 +16,8 @@ pydot
 networkx
 
 tldextract
+ipwhois
+pycountry
 defusedxml
 cryptography
 msgpack
@@ -46,10 +48,7 @@ social-analyzer
 requests_futures
 tweepy
 jellyfish
-shodan
 pandas
-pycountry
-ipwhois
 docker
 email-validator
 docx2python