Bump python to 3.13, and app version

Convert | to _

Another bug fix & bump version

Core/EntityDB.py

@@ -15,9 +15,7 @@ class EntitiesDB:
     links on a project-wide scale.
     """
 
-    def __init__(self, mainWindow, messageHandler, resourceHandler) -> None:
-        self.messageHandler = messageHandler
-        self.resourceHandler = resourceHandler
+    def __init__(self, mainWindow) -> None:
         self.mainWindow = mainWindow
         self.dbLock = Lock()
         self.database = None
@@ -33,17 +31,18 @@ class EntitiesDB:
             if self.database is not None:
                 self.save()
             databaseFile = Path(self.mainWindow.SETTINGS.value("Project/FilesDir")).joinpath("LocalEntitiesDB.lsdb")
-            self.messageHandler.debug(f'Opening Database at: {str(databaseFile)}')
+            self.mainWindow.MESSAGEHANDLER.debug(f'Opening Database at: {databaseFile}')
             try:
                 with open(databaseFile, "rb") as dbFile:
                     self.database = self.mainWindow.RESOURCEHANDLER.reconstructGraphFullFromFile(load(dbFile))
-                self.messageHandler.info('Loaded Local Entities Database.')
+                self.mainWindow.MESSAGEHANDLER.info('Loaded Local Entities Database.')
             except FileNotFoundError:
-                self.messageHandler.info('Creating new Local Entities Database.')
+                self.mainWindow.MESSAGEHANDLER.info('Creating new Local Entities Database.')
                 self.database = nx.DiGraph()
             except Exception as exc:
-                self.messageHandler.error(f'Cannot parse Database: {str(exc)}\nCreating new Local Entities Database.',
-                                          popUp=True)
+                self.mainWindow.MESSAGEHANDLER.error(
+                    f'Cannot parse Database: {exc}\nCreating new Local Entities Database.',
+                    popUp=True)
                 self.database = nx.DiGraph()
 
     def resetTimeline(self) -> None:
@@ -75,7 +74,7 @@ class EntitiesDB:
             with open(tmpSavePath, "wb") as dbFile:
                 dump(self.mainWindow.RESOURCEHANDLER.deconstructGraphForFileDump(self.database), dbFile)
             move(tmpSavePath, databaseFile)
-            self.messageHandler.info('Database Saved.')
+            self.mainWindow.MESSAGEHANDLER.info('Database Saved.')
 
     def addEntity(self, entJson: dict, fromServer: bool = False, updateTimeline: bool = True) -> Union[dict, None]:
         """
@@ -89,7 +88,7 @@ class EntitiesDB:
             if entJson.get('uid') is not None:
                 exists = self.getEntityNoLock(entJson.get('uid'))
 
-            entity = self.resourceHandler.getEntityJson(
+            entity = self.mainWindow.RESOURCEHANDLER.getEntityJson(
                 entJson.get('Entity Type'),
                 entJson)
 
@@ -124,7 +123,7 @@ class EntitiesDB:
                 if entJson.get('uid') is not None:
                     exists = self.getEntityNoLock(entJson.get('uid'))
 
-                entity = self.resourceHandler.getEntityJson(
+                entity = self.mainWindow.RESOURCEHANDLER.getEntityJson(
                     entJson.get('Entity Type'),
                     entJson)
 
@@ -155,13 +154,12 @@ class EntitiesDB:
         """
         with self.dbLock:
             exists = self.isLinkNoLock(linkJson['uid'])
-            link = self.resourceHandler.getLinkJson(linkJson)
+            link = self.mainWindow.RESOURCEHANDLER.getLinkJson(linkJson)
             if link is None:
                 # This can technically be caused by a race condition if the user
                 #   either tries really hard or gets really unlucky.
                 # Caused by deleting a node faster than the link can be created.
-                self.messageHandler.error("Attempted to add Link with "
-                                          "no uid to database.", popUp=True)
+                self.mainWindow.MESSAGEHANDLER.error("Attempted to add Link with no uid to database.", popUp=True)
                 return None
             else:
                 linkUID = link['uid']
@@ -203,7 +201,7 @@ class EntitiesDB:
             try:
                 returnValue = self.database.nodes[uid]
             except KeyError:
-                self.messageHandler.warning(f"Tried to get entity with nonexistent UID: {uid}")
+                self.mainWindow.MESSAGEHANDLER.warning(f"Tried to get entity with nonexistent UID: {uid}")
             finally:
                 return returnValue
 
@@ -215,8 +213,8 @@ class EntitiesDB:
             returnValue = None
             try:
                 returnValue = [self.database.nodes[node] for node in self.database.nodes()]
-            except KeyError:
-                self.messageHandler.error("Tried to get entity with nonexistent UID.")
+            except KeyError as keyError:
+                self.mainWindow.MESSAGEHANDLER.error(f"Tried to get entity with nonexistent UID. Error: {keyError}")
             finally:
                 return returnValue
 
@@ -230,7 +228,7 @@ class EntitiesDB:
             try:
                 returnValue = [self.database.edges[edge] for edge in self.database.edges()]
             except KeyError:
-                self.messageHandler.error("Tried to get link with nonexistent UID.")
+                self.mainWindow.MESSAGEHANDLER.error("Tried to get link with nonexistent UID.")
             finally:
                 return returnValue
 
@@ -258,8 +256,7 @@ class EntitiesDB:
             try:
                 returnValue = self.database.edges[uid]
             except KeyError:
-                self.messageHandler.error(
-                    "Tried to get link with nonexistent UID.")
+                self.mainWindow.MESSAGEHANDLER.error(f"Tried to get link with nonexistent UID: {uid}")
             finally:
                 return returnValue
 
@@ -310,7 +307,7 @@ class EntitiesDB:
         Checks if an entity with the specified primary attribute exists, and if it does, return it.
         """
         result = None
-        primaryField = self.resourceHandler.getPrimaryFieldForEntityType(entityType)
+        primaryField = self.mainWindow.RESOURCEHANDLER.getPrimaryFieldForEntityType(entityType)
         if primaryField is None:
             return result
         with self.dbLock:

Core/FrontendCommunicationsHandler.py

@@ -27,7 +27,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.exceptions import InvalidTag
 
 # Amount of data to place in each message
-MESSAGE_DATA_SIZE = 8192
+MESSAGE_DATA_SIZE = 8192 * 5
 
 # Needs to be a bit bigger than MESSAGE_DATA_SIZE
 RECV_SIZE = MESSAGE_DATA_SIZE + 1024
@@ -149,12 +149,10 @@ class CommunicationsHandler(QtCore.QObject):
             self.sock.send(passMessage)
             messageReceived = decrypter.update(self.sock.recv(RECV_SIZE)) + decrypter.finalize()
             if messageReceived == b"Passphrase is OK":
-                self.threadInc = threading.Thread(target=self.scanIncoming)
-                self.threadInc.setDaemon(True)
+                self.threadInc = threading.Thread(target=self.scanIncoming, daemon=True)
                 self.threadInc.start()
 
-                self.threadInb = threading.Thread(target=self.scanInbox)
-                self.threadInb.setDaemon(True)
+                self.threadInb = threading.Thread(target=self.scanInbox, daemon=True)
                 self.threadInb.start()
 
                 return True
@@ -270,7 +268,7 @@ class CommunicationsHandler(QtCore.QObject):
                     # Socket closed.
                     break
                 receivedInfo = oldData + receivedInfo
-                messages = receivedInfo.split(b'\x03\x03\x03\x03\x03')
+                messages = receivedInfo.split(b'\x03\x03\x03\x03\x03\x03\x03\x03')
                 # Last message is either blank (i.e. '') or incomplete data, so we ignore it.
                 oldData = messages[-1]
                 messages = messages[:-1]
@@ -364,9 +362,7 @@ class CommunicationsHandler(QtCore.QObject):
             with contextlib.suppress(KeyError):
                 dereferenced_entity = dict(entity)
                 resolution_entities_to_send.append(dereferenced_entity)
-                # Icon is not necessary for any resolution as of now: 2022/1/2.
-                # Cutting it out saves data.
-                dereferenced_entity['Icon'] = ''
+                dereferenced_entity['Icon'] = dereferenced_entity['Icon'].toBase64().data()
         message = {'Operation': 'Run Resolution',
                    'Arguments': {
                        'resolution_name': resolution_name,
@@ -382,6 +378,9 @@ class CommunicationsHandler(QtCore.QObject):
             self.receive_completed_resolution_string_result_signal.emit(resolution_name, resolution_result,
                                                                         resolution_uid)
         else:
+            for res_result in resolution_result:
+                if res_icon := res_result[0].get('Icon'):
+                    res_result[0]['Icon'] = QtCore.QByteArray(b64decode(res_icon))
             self.receive_completed_resolution_result_signal.emit(resolution_name, resolution_result,
                                                                  resolution_uid)
         self.remove_server_resolution_from_running_signal.emit(resolution_uid)
@@ -542,8 +541,8 @@ class CommunicationsHandler(QtCore.QObject):
         :param filePath:
         :return:
         """
-        sendHelperThread = threading.Thread(target=self.sendFileHelper, args=(project_name, file_name, filePath))
-        sendHelperThread.setDaemon(True)
+        sendHelperThread = threading.Thread(target=self.sendFileHelper, daemon=True,
+                                            args=(project_name, file_name, filePath))
         self.uploadingFiles[file_name] = sendHelperThread
         sendHelperThread.start()
 
@@ -559,7 +558,7 @@ class CommunicationsHandler(QtCore.QObject):
         if not filePath.exists() or not filePath.is_file():
             return
         with open(filePath, 'rb') as fileHandler:
-            currThread = threading.currentThread()
+            currThread = threading.current_thread()
             while getattr(currThread, "continue_running", True):
                 filePart = fileHandler.read(512)
                 if not filePart:

Core/Interface/CentralPane.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import contextlib
-import re
 import json
 import sys
 import threading
@@ -12,8 +11,6 @@ import folium
 import networkx as nx
 from shutil import move
 from msgpack import dump, load
-from PIL import Image
-from PIL.ImageQt import ImageQt
 from pathlib import Path
 from PySide6 import QtWidgets, QtGui, QtCore
 from PySide6.QtWidgets import QGraphicsPixmapItem
@@ -21,7 +18,7 @@ from PySide6.QtSvgWidgets import QGraphicsSvgItem
 from PySide6.QtWebEngineWidgets import QWebEngineView
 
 from Core.Interface import Entity
-from Core.ResourceHandler import RichNotesEditor
+from Core.ResourceHandler import RichNotesEditor, resizePictureFromBuffer
 from Core.GlobalVariables import hidden_fields
 
 
@@ -193,11 +190,11 @@ class TabbedPane(QtWidgets.QTabWidget):
         self.tabsNotesDict = {}
         self.previousTab = None
 
-        self.allBanners = {bannerID: bannerPath
-                           for bannerID, bannerPath in self.mainWindow.RESOURCEHANDLER.banners.items()}
-
         self.currentChanged.connect(self.currentTabChangedListener)
 
+    def getAllBanners(self) -> dict:
+        return dict(self.mainWindow.RESOURCEHANDLER.banners)
+
     def getCanvasDBPath(self):
         return Path(self.mainWindow.SETTINGS.value("Project/BaseDir")) / "Project Files" / "CanvasTabs.lscanvas"
 
@@ -762,7 +759,6 @@ class CanvasView(QtWidgets.QGraphicsView):
         self.setSizePolicy(QtWidgets.QSizePolicy(
             QtWidgets.QSizePolicy.Policy.Expanding, QtWidgets.QSizePolicy.Policy.Expanding))
 
-        self.dragOver = False
         self.synced = False
 
         self.menu = QtWidgets.QMenu()
@@ -864,7 +860,7 @@ class CanvasView(QtWidgets.QGraphicsView):
     def drawBackground(self, painter: QtGui.QPainter, rect: Union[QtCore.QRectF, QtCore.QRect]) -> None:
         super(CanvasView, self).drawBackground(painter, rect)
         # Ensure that all links will always be drawn.
-        [link.paint(painter, None, None) for link in list(self.scene().linksDict.values())]
+        [link.paint(painter, None, None) for link in self.scene().linksDict.values()]
 
     def centerViewportOnNode(self, uid: str) -> None:
         node = self.scene().getVisibleNodeForUID(uid)
@@ -878,7 +874,6 @@ class CanvasView(QtWidgets.QGraphicsView):
     def dragEnterEvent(self, event) -> None:
         if event.mimeData().hasText() or event.mimeData().hasImage():
             event.setAccepted(True)
-            self.dragOver = True
             self.update()
 
     def dragLeaveEvent(self, event) -> None:
@@ -886,7 +881,6 @@ class CanvasView(QtWidgets.QGraphicsView):
 
     def dropEvent(self, event) -> None:
         pos = self.mapToScene(event.pos())
-        self.dragOver = False
         mimeData = event.mimeData()
         jsonData = None
 
@@ -1167,11 +1161,11 @@ class CanvasView(QtWidgets.QGraphicsView):
 
     def setBanners(self) -> None:
         selectedEntities = [item for item in self.scene().selectedItems() if isinstance(item, Entity.BaseNode)]
-        if len(selectedEntities) < 1:
+        if not selectedEntities:
             self.tabbedPane.mainWindow.MESSAGEHANDLER.warning('Need to select at least one Entity to set its banner.',
                                                               popUp=True)
             return
-        bannerDialog = BannerSelector(self.tabbedPane.allBanners)
+        bannerDialog = BannerSelector(self.tabbedPane.getAllBanners())
         if bannerDialog.exec():
             try:
                 # This following line will throw IndexError if no banner is selected.
@@ -1280,7 +1274,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         """
         if bannerName:
             try:
-                bannerPathStr = self.parent().allBanners[bannerName]
+                bannerPathStr = self.parent().getAllBanners()[bannerName]
                 with open(bannerPathStr, 'rb') as bannerFile:
                     bannerByteArray = QtCore.QByteArray(bannerFile.read())
                 for entity in entities:
@@ -1296,13 +1290,14 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             for entity in entities:
                 try:
                     entityJson = self.parent().mainWindow.LENTDB.getEntity(entity.uid)
-                    bannerPathStr = self.parent().allBanners.get(entityJson.get('Canvas Banner', ''), '')
-                    if not bannerPathStr:
-                        entity.updateBanner(True, None)
-                    else:
+                    if bannerPathStr := self.parent().getAllBanners().get(
+                        entityJson.get('Canvas Banner', ''), ''
+                    ):
                         with open(bannerPathStr, 'rb') as bannerFile:
                             bannerByteArray = QtCore.QByteArray(bannerFile.read())
                         entity.updateBanner(False, bannerByteArray)
+                    else:
+                        entity.updateBanner(True, None)
                 except FileNotFoundError:
                     if bannerName not in notFoundBanners:
                         self.parent().mainWindow.MESSAGEHANDLER.error(f'Banner Icon not found in filesystem: '
@@ -1482,11 +1477,11 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             if groupItems is None:
                 newNode = Entity.BaseNode(picture, node, nodePrimaryAttribute, self.entityTextFont,
                                           self.entityTextBrush)
-                self.addNodeToScene(newNode)
+                self.addNodeToScene(newNode, positions[node][0], positions[node][1])
             else:
                 newNode = Entity.GroupNode(picture, node, nodePrimaryAttribute, self.entityTextFont,
                                            self.entityTextBrush)
-                self.addNodeToScene(newNode)
+                self.addNodeToScene(newNode, positions[node][0], positions[node][1])
 
                 newGroupList = newNode.listWidget
                 newGroupListGraphic = self.addWidget(newGroupList)
@@ -1495,8 +1490,6 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 for item in groupItems:
                     self.sceneGraph.add_node(item, groupID=newNode.uid)
 
-            newNode.setPos(QtCore.QPointF(positions[node][0], positions[node][1]))
-
             progressValue += 1
             progress.setValue(progressValue)
 
@@ -1697,6 +1690,9 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 # Should never happen, but we will handle it if it does.
                 self.parent().mainWindow.MESSAGEHANDLER.warning(f'Entity without valid Date Created: {str(node)}')
                 entityDate = datetime.now().replace(microsecond=0, second=0)
+            if entityDate.tzinfo is None or entityDate.tzinfo.utcoffset(entityDate) is None:
+                # Make timezone-naive objects into timezone-aware, with the user's current timezone.
+                entityDate = entityDate.replace(tzinfo=datetime.now().astimezone().tzinfo)
             if entityDate not in nodesOnCanvas:
                 nodesOnCanvas[entityDate] = [node]
             else:
@@ -1887,6 +1883,7 @@ class CanvasScene(QtWidgets.QGraphicsScene):
             self.removeItem(item.iconItem)
 
             pictureByteArray = pEditor.objectJson['Icon']
+            pictureByteArray = resizePictureFromBuffer(pictureByteArray, (40, 40))
             if pictureByteArray.data().startswith(b'<svg '):
                 item.iconItem = QGraphicsSvgItem()
                 item.iconItem.renderer().load(pictureByteArray)
@@ -1924,9 +1921,16 @@ class CanvasScene(QtWidgets.QGraphicsScene):
         for node in self.nodesDict:
             self.nodesDict[node].setSelected(True)
 
-    def selectChildNodes(self) -> None:
-        items = [item.uid for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
+    def selectNodesFromList(self, nodesList: list):
         self.clearSelection()
+        for node in nodesList:
+            if node in self.nodesDict:
+                self.nodesDict[node].setSelected(True)
+
+    def selectChildNodes(self, clearSelection: bool = True) -> None:
+        items = [item.uid for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
+        if clearSelection:
+            self.clearSelection()
         for item in items:
             childLinks = self.parent().entityDB.getOutgoingLinks(item)
             for childLink in childLinks:
@@ -1934,9 +1938,10 @@ class CanvasScene(QtWidgets.QGraphicsScene):
                 with contextlib.suppress(KeyError):
                     self.nodesDict[childLink[1]].setSelected(True)
 
-    def selectParentNodes(self) -> None:
+    def selectParentNodes(self, clearSelection: bool = True) -> None:
         items = [item.uid for item in self.selectedItems() if isinstance(item, Entity.BaseNode)]
-        self.clearSelection()
+        if clearSelection:
+            self.clearSelection()
         for item in items:
             parentLinks = self.parent().entityDB.getIncomingLinks(item)
             for parentLink in parentLinks:
@@ -2158,10 +2163,11 @@ class PropertiesEditorIconField(QtWidgets.QLabel):
         super(PropertiesEditorIconField, self).__init__()
         self.pictureByteArray = pictureByteArray
         pixmapToSet = QtGui.QPixmap()
-        pixmapToSet.loadFromData(pictureByteArray)
+        pixmapToSet.loadFromData(resizePictureFromBuffer(pictureByteArray, (40, 40)))
         self.setPixmap(pixmapToSet)
         self.uid = uid
         self.canvas = canvas
+        self.mainWindow = self.canvas.parent().mainWindow
 
     def mousePressEvent(self, event: QtGui.QMouseEvent) -> None:
 
@@ -2169,39 +2175,13 @@ class PropertiesEditorIconField(QtWidgets.QLabel):
                                                                options=QtWidgets.QFileDialog.Option.DontUseNativeDialog,
                                                                filter="Image Files (*.png *.jpg *.bmp *.svg)")[0]
         if selectedPath != '':
-            try:
-                filePath = Path(selectedPath)
-
-                with open(filePath, 'rb') as newIconFile:
-                    fileContents = newIconFile.read()
-                if fileContents.startswith(b'<svg '):
-                    widthRegex = re.compile(b' width="\d*" ')
-                    for widthMatches in widthRegex.findall(fileContents):
-                        fileContents = fileContents.replace(widthMatches, b' ')
-                    heightRegex = re.compile(b' height="\d*" ')
-                    for heightMatches in heightRegex.findall(fileContents):
-                        fileContents = fileContents.replace(heightMatches, b' ')
-                    fileContents = fileContents.replace(b'<svg ', b'<svg height="40" width="40" ', 1)
-                    self.pictureByteArray = QtCore.QByteArray(fileContents)
-                else:
-                    image = Image.open(selectedPath)
-                    thumbSize = 40, 40
-                    thumbnail = ImageQt(image.resize(thumbSize))
-                    self.pictureByteArray = QtCore.QByteArray()
-                    imageBuffer = QtCore.QBuffer(self.pictureByteArray)
-
-                    imageBuffer.open(QtCore.QIODevice.OpenModeFlag.WriteOnly)
-
-                    thumbnail.save(imageBuffer, "PNG")
-                    imageBuffer.close()
-
+            filePath = Path(selectedPath)
+            newPic = self.mainWindow.RESOURCEHANDLER.getPictureFromFile(filePath)
+            if newPic is not None:
+                self.pictureByteArray = newPic
                 pixmapToSet = QtGui.QPixmap()
-                pixmapToSet.loadFromData(self.pictureByteArray)
+                pixmapToSet.loadFromData(resizePictureFromBuffer(newPic, (40, 40)))
                 self.setPixmap(pixmapToSet)
-            except ValueError as ve:
-                # Image type is unsupported (for ImageQt)
-                # Supported types: 1, L, P, RGB, RGBA
-                self.canvas.parent().mainWindow.MESSAGEHANDLER.warning(f'Invalid Image selected: {str(ve)}', popUp=True)
 
         super(PropertiesEditorIconField, self).mousePressEvent(event)
 

Core/Interface/DockBarOne.py

@@ -3,6 +3,8 @@ from json import dumps
 from Core.Interface.Entity import BaseNode
 from PySide6 import QtWidgets, QtCore, QtGui
 
+from Core.ResourceHandler import resizePictureFromBuffer
+
 
 class DockBarOne(QtWidgets.QDockWidget):
 
@@ -116,7 +118,8 @@ class EntityList(QtWidgets.QTreeWidget):
             topItem = self.entityTypes[entity['Entity Type']]
             primaryAttr = entity[list(entity)[1]]
             pixmapIcon = QtGui.QPixmap()
-            pixmapIcon.loadFromData(entity['Icon'])
+            resizedIcon = resizePictureFromBuffer(entity['Icon'], (40, 40))
+            pixmapIcon.loadFromData(resizedIcon)
             EntityWidget(topItem,
                          entity['uid'],
                          QtGui.QIcon(pixmapIcon),
@@ -142,7 +145,8 @@ class EntityList(QtWidgets.QTreeWidget):
                 child.setText(0, primaryAttr)
                 return
         pixmapIcon = QtGui.QPixmap()
-        pixmapIcon.loadFromData(entityJson['Icon'])
+        resizedIcon = resizePictureFromBuffer(entityJson['Icon'], (40, 40))
+        pixmapIcon.loadFromData(resizedIcon)
         EntityWidget(entityTypeItem,
                      entityJson['uid'],
                      QtGui.QIcon(pixmapIcon),
@@ -174,8 +178,6 @@ class EntityList(QtWidgets.QTreeWidget):
         """
         Handle dragging of entities onto canvas.
         """
-        super().mouseMoveEvent(event)
-
         if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
             itemDragged = self.itemAt(event.pos())
         else:

Core/Interface/DockBarTwo.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import magic
 from PySide6 import QtWidgets, QtCore, QtGui
-from Core.ResourceHandler import MinSizeStackedLayout, RichNotesEditor
+from Core.ResourceHandler import MinSizeStackedLayout, RichNotesEditor, resizePictureFromBuffer
 from Core.GlobalVariables import hidden_fields_dockbars
 
 
@@ -130,8 +130,10 @@ class EntityDetails(QtWidgets.QWidget):
         self.summaryIcon = QtWidgets.QLabel("")
         self.summaryIcon.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Minimum)
         self.entityTypeLabel = QtWidgets.QLabel("")
-        self.entityUIDLabel = QtWidgets.QLabel("")
-        self.entityPrimaryLabel = QtWidgets.QLabel("")
+        self.entityUIDLabel = QtWidgets.QLineEdit("")
+        self.entityUIDLabel.setReadOnly(True)
+        self.entityPrimaryLabel = QtWidgets.QLineEdit("")
+        self.entityPrimaryLabel.setReadOnly(True)
         summaryLayout.addWidget(self.summaryIcon, 0, 0)
         summaryLayout.addWidget(self.entityTypeLabel, 0, 1, 1, 3)
         summaryLayout.addWidget(self.entityPrimaryLabel, 1, 1)
@@ -281,13 +283,16 @@ class EntityDetails(QtWidgets.QWidget):
                 self.detailsLayoutOneNode.addWidget(notesTextArea, rowCount, 1, 10, 1)
                 rowCount += 9
             else:
+                valueLabel = QtWidgets.QLineEdit(str(jsonDict[key]))
+                valueLabel.setReadOnly(True)
+
                 self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(key), rowCount, 0)
-                self.detailsLayoutOneNode.addWidget(QtWidgets.QLabel(str(jsonDict[key])), rowCount, 1)
+                self.detailsLayoutOneNode.addWidget(valueLabel, rowCount, 1)
             rowCount += 1
         filePath = jsonDict.get('File Path')
         if filePath is not None:
             fullFilePath = Path(self.mainWindow.SETTINGS.value('Project/FilesDir')) / filePath
-            if fullFilePath.exists() and fullFilePath.is_file():
+            if fullFilePath.is_file():
                 magicType = magic.from_file(str(fullFilePath), mime=True)
                 if magicType.split('/')[0] == 'image':
                     previewImage = QtGui.QImage(fullFilePath)
@@ -313,9 +318,9 @@ class EntityDetails(QtWidgets.QWidget):
         second = uid[1]
         secondJson = self.entityDB.getEntity(second)
         firstPixmap = QtGui.QPixmap()
-        firstPixmap.loadFromData(firstJson.get('Icon'))
+        firstPixmap.loadFromData(resizePictureFromBuffer(firstJson.get('Icon'), (40, 40)))
         secondPixmap = QtGui.QPixmap()
-        secondPixmap.loadFromData(secondJson.get('Icon'))
+        secondPixmap.loadFromData(resizePictureFromBuffer(secondJson.get('Icon'), (40, 40)))
         self.linkParent.linkItemPic.setPixmap(firstPixmap)
         self.linkParent.linkItemName.setText(firstJson[list(firstJson)[1]])
         self.linkParent.linkItemUid = firstJson['uid']
@@ -330,7 +335,7 @@ class EntityDetails(QtWidgets.QWidget):
             uid = edge[0]
             edgeJson = self.entityDB.getEntity(uid)
             nodePixmap = QtGui.QPixmap()
-            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            nodePixmap.loadFromData(resizePictureFromBuffer(edgeJson.get('Icon'), (40, 40)))
             ResolutionTreeWidgetEntity(self.relationshipsIncomingTable,
                                        nodePixmap,
                                        edgeJson[list(edgeJson)[1]],
@@ -340,7 +345,7 @@ class EntityDetails(QtWidgets.QWidget):
             uid = edge[1]
             edgeJson = self.entityDB.getEntity(uid)
             nodePixmap = QtGui.QPixmap()
-            nodePixmap.loadFromData(edgeJson.get('Icon'))
+            nodePixmap.loadFromData(resizePictureFromBuffer(edgeJson.get('Icon'), (40, 40)))
             ResolutionTreeWidgetEntity(self.relationshipsOutgoingTable,
                                        nodePixmap,
                                        edgeJson[list(edgeJson)[1]],
@@ -351,7 +356,7 @@ class EntityDetails(QtWidgets.QWidget):
         inc = len(self.entityDB.getIncomingLinks(nodeJson['uid']))
         out = len(self.entityDB.getOutgoingLinks(nodeJson['uid']))
         nodePixmap = QtGui.QPixmap()
-        nodePixmap.loadFromData(nodeJson.get('Icon'))
+        nodePixmap.loadFromData(resizePictureFromBuffer(nodeJson.get('Icon'), (40, 40)))
         ResolutionTreeWidgetEntity(self.nodesTable,
                                    nodePixmap,
                                    nodeJson[list(nodeJson)[1]],
@@ -376,7 +381,7 @@ class EntityDetails(QtWidgets.QWidget):
             self.entityPrimaryLabel.setText(jsonDict[list(jsonDict)[0]])
             self.entityTypeLabel.setText(jsonDict['Entity Type'])
             summaryPixmap = QtGui.QPixmap()
-            summaryPixmap.loadFromData(jsonDict.get('Icon'))
+            summaryPixmap.loadFromData(resizePictureFromBuffer(jsonDict.get('Icon'), (40, 40)))
             self.summaryIcon.setPixmap(summaryPixmap)
         else:
             self.entityUIDLabel.setText("--")
@@ -415,7 +420,8 @@ class SingleLinkItem(QtWidgets.QWidget):
         self.linkItemPic = QtWidgets.QLabel()
 
         self.linkItemPic.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
-        self.linkItemName = QtWidgets.QLabel()
+        self.linkItemName = QtWidgets.QLineEdit()
+        self.linkItemName.setReadOnly(True)
 
         self.linkItemName.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         self.linkItemUid = ""

Core/Interface/Entity.py

@@ -10,6 +10,8 @@ from PySide6.QtWidgets import QGraphicsItem
 from PySide6.QtWidgets import QGraphicsItemGroup, QGraphicsSimpleTextItem, QGraphicsPixmapItem, QGraphicsTextItem
 from PySide6.QtSvgWidgets import QGraphicsSvgItem
 
+from Core.ResourceHandler import resizePictureFromBuffer
+
 ENTITY_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
 LINK_TEXT_FONT = QtGui.QFont("Mono", 11, 700)
 
@@ -21,16 +23,17 @@ class BaseNode(QGraphicsItemGroup):
         super(BaseNode, self).__init__()
 
         self.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+        resizedByteArray = resizePictureFromBuffer(pictureByteArray, (40, 40))
 
-        if pictureByteArray.data().startswith(b'<svg '):
+        if pictureByteArray.data().startswith(b'<svg ') or pictureByteArray.data().startswith(b'<?xml'):
             self.iconItem = QGraphicsSvgItem()
-            self.iconItem.renderer().load(pictureByteArray)
+            self.iconItem.renderer().load(resizedByteArray)
             # Force recalculation of geometry, else this looks like 1 pixel.
             # https://stackoverflow.com/a/68182093
             self.iconItem.setElementId("")
         else:
             pixmapItem = QtGui.QPixmap()
-            pixmapItem.loadFromData(pictureByteArray)
+            pixmapItem.loadFromData(resizedByteArray)
             self.iconItem = QGraphicsPixmapItem(pixmapItem)
 
         self.labelItem = QGraphicsTextItem('')
@@ -180,7 +183,7 @@ class GroupNode(BaseNode):
             except IndexError:
                 primaryField = ''
             iconPixmap = QtGui.QPixmap()
-            iconPixmap.loadFromData(entityJson['Icon'])
+            iconPixmap.loadFromData(resizePictureFromBuffer(entityJson['Icon'], (40, 40)))
 
             GroupNodeListItem(icon=iconPixmap, text=primaryField, uid=uid,
                               listview=self.listWidget.itemList)
@@ -203,6 +206,7 @@ class GroupNode(BaseNode):
         [self.addItemToGroup(uid) for uid in childNodeUIDs]  # Should be faster than just a for loop
         self.listProxyWidget = listProxyWidget
         self.listProxyWidget.setCacheMode(QGraphicsItemGroup.CacheMode.DeviceCoordinateCache)
+        self.listProxyWidget.setZValue(100)
 
     def addItemToGroup(self, uid: str) -> None:
         self.groupedNodesUid.add(uid)
@@ -298,10 +302,6 @@ class BaseConnector(QGraphicsItemGroup):
     def endItem(self) -> BaseNode:
         return self.myEndItem
 
-    def updatePosition(self) -> None:
-        self.line = QtCore.QLineF(self.mapFromItem(self.myStartItem, 0, 0), self.mapFromItem(self.myEndItem, 0, 0))
-        self.update()
-
     def boundingRect(self) -> QtCore.QRectF:
         extra = self.pen.width() + 20
         p1 = self.line.p1()
@@ -318,8 +318,8 @@ class BaseConnector(QGraphicsItemGroup):
     def paint(self, painter: QtGui.QPainter, option: QtWidgets.QStyleOptionGraphicsItem,
               widget: Optional[QtWidgets.QWidget] = ...) -> None:
 
-        currentStartPos = self.myStartItem.pos()
-        currentEndPos = self.myEndItem.pos()
+        currentStartPos = self.myStartItem.pos() - self.pos()
+        currentEndPos = self.myEndItem.pos() - self.pos()
 
         self.myColor = self.colorSelected if self.isSelected() else self.colorDefault
 

Core/Interface/MenuBar.py

@@ -29,7 +29,7 @@ from PySide6 import QtWidgets, QtGui, QtCore
 from Core.GlobalVariables import user_agents
 from Core.Interface.Entity import BaseNode
 from Core.ResourceHandler import StringPropertyInput, FilePropertyInput, SingleChoicePropertyInput, \
-    MultiChoicePropertyInput
+    MultiChoicePropertyInput, resizePictureFromBuffer
 
 
 class MenuBar(QtWidgets.QMenuBar):
@@ -61,6 +61,18 @@ class MenuBar(QtWidgets.QMenuBar):
                                      triggered=self.rename)
         fileMenu.addAction(renameAction)
 
+        openProjectFilesAction = QtGui.QAction("Browse Project Files",
+                                     self,
+                                     statusTip="Open the Project Files directory for this project.",
+                                     triggered=self.openProjectFilesDir)
+        fileMenu.addAction(openProjectFilesAction)
+
+        checkForUpdateAction = QtGui.QAction("Check for Updates",
+                                     self,
+                                     statusTip="Check if there are any updates for LinkScope available.",
+                                     triggered=self.checkForUpdates)
+        fileMenu.addAction(checkForUpdateAction)
+
         importMenu = self.addMenu("Import")
 
         fromBrowserAction = QtGui.QAction("From Browser",
@@ -253,6 +265,14 @@ class MenuBar(QtWidgets.QMenuBar):
         actionSelectChildren.setShortcut('Ctrl+Shift+C')
         nodeOperationsMenu.addAction(actionSelectChildren)
 
+        actionExpandSelectChildren = QtGui.QAction('Include Child Nodes in Selection',
+                                                   self,
+                                                   statusTip="Include the child entities of the selected nodes "
+                                                             "in the current selection.",
+                                                   triggered=self.selectExpandChildNodes)
+        actionExpandSelectChildren.setShortcut('Ctrl+Alt+C')
+        nodeOperationsMenu.addAction(actionExpandSelectChildren)
+
         actionSelectParents = QtGui.QAction('Select Parent Nodes',
                                             self,
                                             statusTip="Select the parent entities of the selected nodes.",
@@ -260,6 +280,14 @@ class MenuBar(QtWidgets.QMenuBar):
         actionSelectParents.setShortcut('Ctrl+Shift+P')
         nodeOperationsMenu.addAction(actionSelectParents)
 
+        actionExpandSelectParents = QtGui.QAction('Include Parent Nodes in Selection',
+                                                  self,
+                                                  statusTip="Include the parent entities of the selected nodes "
+                                                            "in the current selection.",
+                                                  triggered=self.selectExpandParentNodes)
+        actionExpandSelectParents.setShortcut('Ctrl+Alt+P')
+        nodeOperationsMenu.addAction(actionExpandSelectParents)
+
         nodeOperationsMenu.addSeparator()
 
         actionMacrosWizard = QtGui.QAction('Macros...',
@@ -358,6 +386,16 @@ class MenuBar(QtWidgets.QMenuBar):
                                             triggered=self.reloadModules)
         modulesMenu.addAction(reloadModulesAction)
 
+        viewModuleSourcesAction = QtGui.QAction("View Sources", self,
+                                                statusTip="Show the Module Sources Manager",
+                                                triggered=self.viewModuleSources)
+        modulesMenu.addAction(viewModuleSourcesAction)
+
+        viewModuleManagerAction = QtGui.QAction("View Module Manager", self,
+                                                statusTip="Show the Modules Manager",
+                                                triggered=self.viewModuleManager)
+        modulesMenu.addAction(viewModuleManagerAction)
+
         serverMenu = self.addMenu("&Server")
 
         connectAction = QtGui.QAction("Connect", self,
@@ -498,7 +536,7 @@ class MenuBar(QtWidgets.QMenuBar):
         newLinks = []
         fileDirectory = Path(importDialog.fileDirectoryLine.text())
         if importDialogAccept and fileDirectory != '':
-            if fileDirectory.exists() and fileDirectory.is_file():
+            if fileDirectory.is_file():
                 sceneToAddTo = None
 
                 try:
@@ -560,9 +598,7 @@ class MenuBar(QtWidgets.QMenuBar):
 
                         importEntityCSVDialog = ImportEntityFromCSVFile(self, csvDF)
                         if importEntityCSVDialog.exec_():
-                            attributeRows = [comboBox.currentText()
-                                             if comboBox.currentText() else
-                                             csvDF.columns[index]
+                            attributeRows = [comboBox.currentText() or csvDF.columns[index]
                                              for index, comboBox in
                                              enumerate(importEntityCSVDialog.fieldMappingComboBoxes)]
 
@@ -631,9 +667,7 @@ class MenuBar(QtWidgets.QMenuBar):
                                     entityOneType = importLinksCSVDialog.entityOneTypeChoiceDropdown.currentText()
                                     entityTwoType = importLinksCSVDialog.entityTwoTypeChoiceDropdown.currentText()
 
-                                    attributeRows = [comboBox.currentText()
-                                                     if comboBox.currentText()
-                                                     else fieldsRemainingDF.columns[index]
+                                    attributeRows = [comboBox.currentText() or fieldsRemainingDF.columns[index]
                                                      for index, comboBox
                                                      in enumerate(createLinkEntitiesDialog.fieldMappingComboBoxes)]
 
@@ -829,6 +863,12 @@ class MenuBar(QtWidgets.QMenuBar):
     def rename(self) -> None:
         self.parent().renameProjectPromptName()
 
+    def openProjectFilesDir(self) -> None:
+        self.parent().openDirectoryInNativeFileBrowser(self.parent().SETTINGS.value("Project/FilesDir"))
+
+    def checkForUpdates(self) -> None:
+        self.parent().openUpdateWindow()
+
     def editSettings(self) -> None:
         self.parent().editSettings()
 
@@ -853,6 +893,12 @@ class MenuBar(QtWidgets.QMenuBar):
     def reloadModules(self) -> None:
         self.parent().reloadModules()
 
+    def viewModuleSources(self) -> None:
+        self.parent().MODULEMANAGER.showSourcesManager()
+
+    def viewModuleManager(self) -> None:
+        self.parent().MODULEMANAGER.showModuleManager()
+
     def runningResolutions(self) -> None:
         self.parent().cleanUpLocalFinishedResolutions()
         runningResolutionsDialog = ViewAndStopResolutionsDialog(self.parent())
@@ -974,9 +1020,15 @@ class MenuBar(QtWidgets.QMenuBar):
     def selectChildNodes(self) -> None:
         self.parent().centralWidget().tabbedPane.getCurrentScene().selectChildNodes()
 
+    def selectExpandChildNodes(self) -> None:
+        self.parent().centralWidget().tabbedPane.getCurrentScene().selectChildNodes(clearSelection=False)
+
     def selectParentNodes(self) -> None:
         self.parent().centralWidget().tabbedPane.getCurrentScene().selectParentNodes()
 
+    def selectExpandParentNodes(self) -> None:
+        self.parent().centralWidget().tabbedPane.getCurrentScene().selectParentNodes(clearSelection=False)
+
     def viewMacrosWizard(self) -> None:
         self.parent().showMacrosDialog()
 
@@ -1137,7 +1189,7 @@ class MenuBar(QtWidgets.QMenuBar):
                 if itemJSONNotes != '':
                     itemPrimaryField = itemJSON[self.parent().RESOURCEHANDLER.getPrimaryFieldForEntityType(
                         itemJSON['Entity Type'])]
-                    fileName = itemPrimaryField + ' | ' + itemJSON.get('Date Last Edited', str(time.time_ns())) + '.txt'
+                    fileName = itemPrimaryField + '_' + itemJSON.get('Date Last Edited', str(time.time_ns())) + '.txt'
                     fileName = fileName.replace('/', '+')
                     fileName = fileName.replace('\\', '+')
                     with open(baseFilesPath / fileName, "w") as f:
@@ -1623,7 +1675,7 @@ class CollectorsDialog(QtWidgets.QDialog):
         self.baseLayout.addWidget(closeButton)
 
     def startSelectedCollector(self, collectorToStartDict: dict):
-        newCollector = CollectorStartDialog(self.mainWindow.LENTDB, collectorToStartDict)
+        newCollector = CollectorStartDialog(self.mainWindow, collectorToStartDict)
 
         if newCollector.exec_():
             collector_name = collectorToStartDict['name']
@@ -1653,9 +1705,9 @@ class CollectorsDialog(QtWidgets.QDialog):
 
 class CollectorStartDialog(QtWidgets.QDialog):
 
-    def __init__(self, entityDB, collectorDict: dict):
+    def __init__(self, mainWindow, collectorDict: dict):
         super(CollectorStartDialog, self).__init__()
-        self.entityDB = entityDB
+        self.mainWindow = mainWindow
         self.setModal(True)
         self.setWindowTitle('Collector Wizard')
         self.parametersList = []
@@ -1690,12 +1742,13 @@ class CollectorStartDialog(QtWidgets.QDialog):
         self.entitySelector.header().setStretchLastSection(False)
         self.entitySelector.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeMode.Stretch)
         relevantEntityFields = [(entity['uid'], entity[list(entity)[1]], entity['Entity Type'], entity['Icon'])
-                                for entity in entityDB.getAllEntities()
+                                for entity in self.mainWindow.LENTDB.getAllEntities()
                                 if entity['Entity Type'] in originTypes or '*' in originTypes]
         for eligibleEntity in relevantEntityFields:
             newTreeWidgetItem = QtWidgets.QTreeWidgetItem(self.entitySelector)
             newTreeWidgetItemPixmap = QtGui.QPixmap()
-            newTreeWidgetItemPixmap.loadFromData(eligibleEntity[3])
+            resizedIcon = resizePictureFromBuffer(eligibleEntity[3], (40, 40))
+            newTreeWidgetItemPixmap.loadFromData(resizedIcon)
             newTreeWidgetItem.setText(0, eligibleEntity[1])
             newTreeWidgetItem.setText(1, eligibleEntity[2])
             newTreeWidgetItem.setIcon(2, newTreeWidgetItemPixmap)
@@ -1772,7 +1825,7 @@ class CollectorStartDialog(QtWidgets.QDialog):
 
     def accept(self) -> None:
         for item in self.entitySelector.selectedItems():
-            self.chosenItems.append(self.entityDB.getEntity(item.text(3)))
+            self.chosenItems.append(self.mainWindow.LENTDB.getEntity(item.text(3)))
         for resolutionParameterName, resolutionParameterInput in self.parametersList:
             value = resolutionParameterInput.getValue()
             if value == '':
@@ -2154,8 +2207,8 @@ class ImportEntityFromCSVFile(QtWidgets.QDialog):
         if primaryFieldMapped:
             self.accept()
         else:
-            self.parent().parent().MESSAGEHANDLER.warning('Primary field (' + primaryField +
-                                                          ') needs to be mapped before proceeding.', popUp=True)
+            self.parent().parent().MESSAGEHANDLER.warning(
+                f'Primary field ({primaryField}) needs to be mapped before proceeding.', popUp=True)
 
 
 class ImportFromFileDialog(QtWidgets.QDialog):
@@ -2475,7 +2528,7 @@ class ScreenshotWebsiteThread(QtCore.QThread):
         newNodes = []
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=self.mainWindow.getPlaywrightBrowserPath('firefox'))
 
             if platform.system() == 'Linux':
                 context = browser.new_context(
@@ -2568,7 +2621,7 @@ class SaveWebsiteThread(QtCore.QThread):
                         fileToWrite.write(response.body())
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=self.mainWindow.getPlaywrightBrowserPath('firefox'))
 
             if platform.system() == 'Linux':
                 context = browser.new_context(
@@ -2646,7 +2699,7 @@ class ImportBrowserTabsThread(QtCore.QThread):
             if self.importDialog.firefoxChoice.isChecked():
                 recordSession = self.importDialog.firefoxSessionChoice.isChecked()
                 try:
-                    browser = p.firefox.launch()
+                    browser = p.firefox.launch(executable_path=self.mainWindow.getPlaywrightBrowserPath('firefox'))
 
                     if platform.system() == 'Linux':
                         context = browser.new_context(
@@ -2803,7 +2856,7 @@ class ImportBrowserTabsThread(QtCore.QThread):
                 self.progressSignal.emit(progressValue)
 
                 try:
-                    browser = p.chromium.launch()
+                    browser = p.chromium.launch(executable_path=self.mainWindow.getPlaywrightBrowserPath('chromium'))
 
                     # NOTE: Cookies are not obtained for chromium based browsers.
 
@@ -2835,7 +2888,7 @@ class ImportBrowserTabsThread(QtCore.QThread):
                                 latestTimestamp = max([int(sessionFile.split("Session_", 1)[1])
                                                        for sessionFile in os.listdir(sessionFilePath)
                                                        if 'Session_' in sessionFile])
-                                sessionFilePath = sessionFilePath.joinpath("Session_" + str(latestTimestamp))
+                                sessionFilePath = sessionFilePath.joinpath(f"Session_{latestTimestamp}")
                                 chromeSessionFileContents = sessionFilePath.read_bytes()
                                 break
                             except (FileNotFoundError, IndexError):

Core/ReportGeneration.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
 
+import tempfile
+import re
+from shutil import rmtree
+from svglib.svglib import svg2rlg
+from uuid import uuid4
 from datetime import datetime
 from pathlib import Path
 from reportlab.lib import colors
@@ -15,10 +20,369 @@ from reportlab.lib.enums import TA_LEFT, TA_CENTER, TA_JUSTIFY
 from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
 from reportlab.lib.pagesizes import LETTER, inch
 from reportlab.graphics.shapes import Line, Drawing
+from PySide6 import QtWidgets
 
+from Core.Interface.Entity import BaseNode
 from Core.GlobalVariables import avoid_parsing_fields
 
 
+class ReportWizard(QtWidgets.QWizard):
+    def __init__(self, parent):
+        super(ReportWizard, self).__init__(parent=parent)
+        self.reportTempFolder = tempfile.mkdtemp()
+
+        self.primaryFieldsList = []
+
+        self.addPage(InitialConfigPage(self))
+        self.addPage(TitlePage(self))
+
+        self.addPage(SummaryPage(self))
+
+        self.selectedNodes = [entity for entity in
+                              self.parent().centralWidget().tabbedPane.getCurrentScene().selectedItems()
+                              if isinstance(entity, BaseNode)]
+        for selectedNode in self.selectedNodes:
+            # used in wizard
+            self.primaryField = selectedNode.labelItem.toPlainText()
+            self.uid = selectedNode.uid
+
+            # used in report generation
+            self.primaryFieldsList.append(self.primaryField)
+            self.addPage(EntityPage(self))
+
+        self.setWizardStyle(QtWidgets.QWizard.WizardStyle.ModernStyle)
+        self.setWindowTitle("Generate Report Wizard")
+
+        self.button(QtWidgets.QWizard.WizardButton.FinishButton).clicked.connect(self.onFinish)
+
+    def onFinish(self):
+        outgoingEntitiesForEachEntity = []
+        incomingEntitiesForEachEntity = []
+        outgoingEntityPrimaryFieldsForEachEntity = []
+        incomingEntityPrimaryFieldsForEachEntity = []
+
+        entityList = []
+        reportData = []
+        for pageID in self.pageIds():
+            pageObject = self.page(pageID)
+            reportData.append(pageObject.getData())
+
+        for selectedNode in self.selectedNodes:
+            uid = selectedNode.uid
+            entityList.append(self.parent().LENTDB.getEntity(uid))
+            outgoing = self.parent().LENTDB.getOutgoingLinks(uid)
+            incoming = self.parent().LENTDB.getIncomingLinks(uid)
+
+            outgoingEntities = []
+            incomingEntities = []
+            outgoingNames = []
+            incomingNames = []
+
+            for out in outgoing:
+                outLink = self.parent().LENTDB.getLink(out)
+                outgoingEntities.append(outLink)
+                outgoingEntityJson = self.parent().LENTDB.getEntity(outLink['uid'][1])
+                outgoingNames.append(outgoingEntityJson[list(outgoingEntityJson)[1]])
+
+            for inc in incoming:
+                inLink = self.parent().LENTDB.getLink(inc)
+                incomingEntities.append(inLink)
+                incomingEntityJson = self.parent().LENTDB.getEntity(inLink['uid'][0])
+
+                incomingNames.append(incomingEntityJson[list(incomingEntityJson)[1]])
+
+            outgoingEntityPrimaryFieldsForEachEntity.append(outgoingNames)
+            incomingEntityPrimaryFieldsForEachEntity.append(incomingNames)
+            outgoingEntitiesForEachEntity.append(outgoingEntities)
+            incomingEntitiesForEachEntity.append(incomingEntities)
+
+        path = Path(reportData[0].get('SavePath'))
+
+        canvasName = reportData[2].get('CanvasName')
+        viewPortBool = reportData[2].get('ViewPort')
+
+        canvasPicture = self.parent().getPictureOfCanvas(canvasName, viewPortBool, True)
+        canvasImagePath = Path(self.reportTempFolder) / 'canvas.png'
+        canvasPicture.save(str(canvasImagePath), "PNG")
+
+        # timelinePicture = self.parent().dockbarThree.timeWidget.takePictureOfView(False)
+        # timelineImagePath = Path(temp_dir.name) / 'timeline.png'
+        # timelinePicture.save(str(timelineImagePath), "PNG")
+
+        savePath = Path(reportData[0]['SavePath']).absolute()
+
+        try:
+            PDFReport(str(path), reportData, outgoingEntitiesForEachEntity,
+                      incomingEntitiesForEachEntity, entityList, canvasImagePath, None,  # <timelinePic
+                      self.primaryFieldsList, incomingEntityPrimaryFieldsForEachEntity,
+                      outgoingEntityPrimaryFieldsForEachEntity)
+
+            self.parent().MESSAGEHANDLER.debug(reportData)
+            self.parent().MESSAGEHANDLER.info(
+                f"Saved Report at: {str(savePath)}", popUp=True
+            )
+        except PermissionError:
+            self.parent().MESSAGEHANDLER.error(
+                f"Could not generate report. No permission to save at the chosen location: {str(savePath)}",
+                popUp=True,
+                exc_info=False,
+            )
+        except Exception as exc:
+            self.parent().MESSAGEHANDLER.error(
+                f"Could not generate report: {str(exc)}", popUp=True, exc_info=True
+            )
+        finally:
+            rmtree(self.reportTempFolder)
+
+
+class InitialConfigPage(QtWidgets.QWizardPage):
+    def __init__(self, parent=None):
+        super(InitialConfigPage, self).__init__(parent)
+        self.subtitleLabel = QtWidgets.QLabel("Path to save the report at: ")
+        self.savePathEdit = QtWidgets.QLineEdit()
+        self.setTitle("Initial Configuration Wizard")
+
+        pDirButton = QtWidgets.QPushButton("Save Report As...")
+        pDirButton.clicked.connect(self.editPath)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(self.subtitleLabel)
+        hLayout.addWidget(self.savePathEdit)
+        hLayout.addWidget(pDirButton)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def editPath(self):
+        selectedPath = QtWidgets.QFileDialog.getSaveFileName(self,
+                                                             "File Name to Save As",
+                                                             str(Path.home()),
+                                                             filter="PDF Files (*.pdf)",
+                                                             options=QtWidgets.QFileDialog.Option.DontUseNativeDialog)
+        selectedPath = selectedPath[0]
+        if selectedPath != '':
+            savePath = Path(selectedPath).absolute()
+            if savePath.suffix != '.pdf':
+                savePath = savePath.with_suffix(f"{savePath.suffix}.pdf")
+            self.savePathEdit.setText(str(savePath))
+
+    def getData(self):
+        return {'SavePath': self.savePathEdit.text()}
+
+
+class TitlePage(QtWidgets.QWizardPage):
+    def __init__(self, parent=None):
+        super(TitlePage, self).__init__(parent)
+        self.inputTitleEdit = QtWidgets.QLineEdit()
+        self.inputSubtitleEdit = QtWidgets.QLineEdit()
+        self.inputAuthorsEdit = QtWidgets.QLineEdit()
+        self.setTitle("Title Page Wizard")
+
+        titleLabel = QtWidgets.QLabel("Title: ")
+        subtitleLabel = QtWidgets.QLabel("Subtitle: ")
+        authorsLabel = QtWidgets.QLabel("Authors: ")
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(titleLabel)
+        hLayout.addWidget(self.inputTitleEdit)
+        hLayout.addWidget(subtitleLabel)
+        hLayout.addWidget(self.inputSubtitleEdit)
+        hLayout.addWidget(authorsLabel)
+        hLayout.addWidget(self.inputAuthorsEdit)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def getData(self):
+        return {
+            'Title': self.inputTitleEdit.text(),
+            'Subtitle': self.inputSubtitleEdit.text(),
+            'Authors': self.inputAuthorsEdit.text(),
+        }
+
+
+class SummaryPage(QtWidgets.QWizardPage):
+    def __init__(self, parent):
+        super(SummaryPage, self).__init__(parent=parent.parent())
+        self.setTitle("Summary Page Wizard")
+        self.inputNotesEdit = QtWidgets.QPlainTextEdit()
+        self.canvasDropDownMenu = QtWidgets.QComboBox()
+        self.viewPortCheckBox = QtWidgets.QCheckBox('ViewPort Only')
+        self.viewPortCheckBox.setChecked(False)
+        self.canvasNames = list(self.parent().centralWidget().tabbedPane.canvasTabs.keys())
+
+        summaryLabel = QtWidgets.QLabel("Summary Notes: ")
+
+        canvasLabel = QtWidgets.QLabel("Select canvas to be displayed: ")
+        for canvasName in self.canvasNames:
+            self.canvasDropDownMenu.addItem(canvasName)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(summaryLabel)
+        hLayout.addWidget(self.inputNotesEdit)
+        hLayout.addWidget(canvasLabel)
+        hLayout.addWidget(self.viewPortCheckBox)
+        hLayout.addWidget(self.canvasDropDownMenu)
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        self.setLayout(layout)
+
+    def getData(self):
+        return {
+            'SummaryNotes': self.inputNotesEdit.toPlainText(),
+            'CanvasName': self.canvasDropDownMenu.currentText(),
+            'ViewPort': self.viewPortCheckBox.isChecked(),
+        }
+
+
+class EntityPage(QtWidgets.QWizardPage):
+    def __init__(self, parent: ReportWizard):
+        super(EntityPage, self).__init__(parent=parent.parent())
+        self.reportWizard = parent
+
+        self.setTitle("Entity Page Wizard")
+        self.setMinimumSize(300, 700)
+
+        self.entityName = parent.primaryField
+        self.entityUID = parent.uid
+
+        self.inputNotesEdit = QtWidgets.QPlainTextEdit()
+        self.inputImageEdit = QtWidgets.QLineEdit()
+        self.inputImageEdit.setReadOnly(True)
+        self.addAppendixButton = QtWidgets.QPushButton("Add New Appendix Section")
+        self.removeAppendixButton = QtWidgets.QPushButton("Remove Last Appendix Section")
+
+        self.scrolllayout = QtWidgets.QVBoxLayout()
+        self.scrollwidget = QtWidgets.QWidget()
+
+        self.defaultpic = self.parent().LENTDB.getEntity(self.entityUID).get('Icon')
+
+        summaryLabel = QtWidgets.QLabel(f"Entity {self.entityName} Notes: ")
+
+        imageLabel = QtWidgets.QLabel("Image Path: ")
+        pDirButton = QtWidgets.QPushButton("Select Image...")
+        pDirButton.clicked.connect(self.editPath)
+        pDirButton.setDisabled(True)
+        self.imageCheckBox = QtWidgets.QCheckBox('Add Custom Entity Image')
+        self.imageCheckBox.setChecked(False)
+        self.imageCheckBox.toggled.connect(pDirButton.setEnabled)
+
+        self.addAppendixButton.clicked.connect(self.addSection)
+        self.removeAppendixButton.clicked.connect(self.removeSection)
+
+        self.scrollwidget.setLayout(self.scrolllayout)
+
+        scroll = QtWidgets.QScrollArea()
+        scroll.setWidgetResizable(True)
+        scroll.setWidget(self.scrollwidget)
+
+        hLayout = QtWidgets.QVBoxLayout()
+        hLayout.addWidget(summaryLabel)
+        hLayout.addWidget(self.inputNotesEdit)
+
+        hLayout.addWidget(self.imageCheckBox)
+        hLayout.addWidget(imageLabel)
+        hLayout.addWidget(self.inputImageEdit)
+        hLayout.addWidget(pDirButton)
+
+        hLayout.addItem(QtWidgets.QSpacerItem(10, 30))
+        hLayout.addWidget(self.addAppendixButton)
+        hLayout.addWidget(self.removeAppendixButton)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addLayout(hLayout)
+        layout.addWidget(scroll)
+
+        self.setLayout(layout)
+
+    def editPath(self) -> None:
+        selectedPath = QtWidgets.QFileDialog().getOpenFileName(parent=self, caption='Select New Icon',
+                                                               dir=str(Path.home()),
+                                                               options=QtWidgets.QFileDialog.Option.DontUseNativeDialog,
+                                                               filter="Image Files (*.png *.jpg)")[0]
+        if selectedPath != '':
+            self.inputImageEdit.setText(str(Path(selectedPath).absolute()))
+
+    def addSection(self) -> None:
+        appendixWidget = AppendixWidget()
+        self.scrolllayout.addWidget(appendixWidget)
+
+    def removeSection(self) -> None:
+        if numChildren := self.scrolllayout.count():
+            appendixItem = self.scrolllayout.takeAt(numChildren - 1)
+            appendixItem.widget().deleteLater()
+
+    def getData(self):
+        appendixNotes = []
+        if self.inputImageEdit.text() != '' and self.imageCheckBox.isChecked():
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': self.inputImageEdit.text()}
+        elif 'PNG' in str(self.defaultpic):
+            imagePath = Path(self.reportWizard.reportTempFolder) / f'{str(uuid4())}.png'
+            with open(imagePath, 'wb') as tempFile:
+                tempFile.write(bytearray(self.defaultpic.data()))
+
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': str(imagePath)}
+        else:
+            if 'svg' not in str(self.defaultpic):
+                # Default picture is an SVG.
+                self.defaultpic = self.reportWizard.parent().RESOURCEHANDLER.getEntityDefaultPicture(
+                    self.reportWizard.parent().LENTDB.getEntity(self.entityUID)['Entity Type'])
+            contents = bytearray(self.defaultpic)
+            widthRegex = re.compile(b' width="\d*" ')
+            for widthMatches in widthRegex.findall(self.defaultpic):
+                contents = contents.replace(widthMatches, b' ')
+            heightRegex = re.compile(b' height="\d*" ')
+            for heightMatches in heightRegex.findall(self.defaultpic):
+                contents = contents.replace(heightMatches, b' ')
+            contents = contents.replace(b'<svg ', b'<svg height="150" width="150" ')
+
+            imagePath = Path(self.reportWizard.reportTempFolder) / f'{str(uuid4())}.svg'
+            with open(imagePath, 'wb') as tempFile:
+                tempFile.write(contents)
+
+            image = svg2rlg(imagePath)
+            data = {'EntityNotes': self.inputNotesEdit.toPlainText(), 'EntityImage': image}
+
+        for index in range(self.scrolllayout.count()):
+            childWidget = self.scrolllayout.itemAt(index).widget()
+            appendixDict = {'AppendixEntityNotes': childWidget.inputAppendixNotesEdit.toPlainText(),
+                            'AppendixEntityImage': childWidget.inputAppendixImageEdit.text()}
+            appendixNotes.append(appendixDict)
+        return data, appendixNotes
+
+
+class AppendixWidget(QtWidgets.QWidget):
+
+    def __init__(self) -> None:
+        super(AppendixWidget, self).__init__()
+        appendixWidgetLayout = QtWidgets.QGridLayout()
+        appendixLabelNotes = QtWidgets.QLabel("Entity Notes: ")
+        self.inputAppendixNotesEdit = QtWidgets.QPlainTextEdit()
+        imageAppendixLabel = QtWidgets.QLabel("Image Path: ")
+        appendixButton = QtWidgets.QPushButton("Select Image...")
+        appendixButton.clicked.connect(self.editAppendixPath)
+        self.inputAppendixImageEdit = QtWidgets.QLineEdit()
+        self.inputAppendixImageEdit.setReadOnly(True)
+        appendixWidgetLayout.addWidget(appendixLabelNotes, 0, 0, 1, 1)
+        appendixWidgetLayout.addWidget(self.inputAppendixNotesEdit, 2, 0, 4, 1)
+        appendixWidgetLayout.addWidget(imageAppendixLabel, 7, 0, 1, 1)
+        appendixWidgetLayout.addWidget(self.inputAppendixImageEdit, 9, 0, 1, 1)
+        appendixWidgetLayout.addWidget(appendixButton, 11, 0, 1, 1)
+        self.setLayout(appendixWidgetLayout)
+        self.inputAppendixNotesEdit.setFixedHeight(100)
+
+    def editAppendixPath(self) -> None:
+        selectedPath = QtWidgets.QFileDialog().getOpenFileName(parent=self, caption='Select New Icon',
+                                                               dir=str(Path.home()),
+                                                               options=QtWidgets.QFileDialog.Option.DontUseNativeDialog,
+                                                               filter="Image Files (*.png *.jpg)")[0]
+
+        if selectedPath != '':
+            self.inputAppendixImageEdit.setText(str(Path(selectedPath).absolute()))
+
+
 class MyDocTemplate(BaseDocTemplate):
     def __init__(self, filename, **kw):
         self.allowSplitting = 0
@@ -206,14 +570,12 @@ class PDFReport:
                 outgoing_data.append([linkName, childNode, dateCreated, Paragraph(linkNotes)])
             outgoing_table = Table(data=outgoing_data, style=links_table_style, hAlign="CENTER",
                                    colWidths=[140, 140, 140, 140])
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
         else:
             outgoing_data = [
                 ['No Outgoing Links']]
             outgoing_table = Table(data=outgoing_data, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
 
         # parent UID
         if incomingLinks:
@@ -236,14 +598,12 @@ class PDFReport:
                 incoming_data.append([linkName, parentNode, dateCreated, Paragraph(linkNotes)])
             incoming_table = Table(data=incoming_data, style=links_table_style, hAlign="CENTER",
                                    colWidths=[140, 140, 140, 140])
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
         else:
             incoming_data = [
                 ['No Incoming Links']]
             incoming_table = Table(data=incoming_data, hAlign="CENTER")
-            spacer = Spacer(10, 10)
-            self.elements.append(spacer)
+        spacer = Spacer(10, 10)
+        self.elements.append(spacer)
 
         tableParagraph = ParagraphStyle('Report', fontSize=9, justifyBreaks=1, alignment=TA_CENTER,
                                         justifyLastLine=0)
@@ -296,7 +656,7 @@ class PDFReport:
             pie.y = 65
             pie.data = [len(incomingLinks), len(outgoingLinks)]
             pie.sideLabels = 1
-            pie.labels = ['Incoming: ' + str(len(incomingLinks)), 'Outgoing: ' + str(len(outgoingLinks))]
+            pie.labels = [f'Incoming: {len(incomingLinks)}', f'Outgoing: {len(outgoingLinks)}']
             pie.slices.strokeWidth = 1
             if len(incomingLinks) > len(outgoingLinks):
                 pie.slices[0].popout = 5

Core/ResolutionManager.py

@@ -6,20 +6,27 @@ import sys
 from os import listdir
 from pathlib import Path
 from uuid import uuid4
-from typing import Union
+from typing import Union, Any
+
+from PySide6 import QtCore, QtWidgets, QtGui
+from Core.ResourceHandler import StringPropertyInput, FilePropertyInput, SingleChoicePropertyInput, \
+    MultiChoicePropertyInput
 
 
 class ResolutionManager:
 
     # Load all resources needed.
-    def __init__(self, mainWindow, messageHandler):
-        self.messageHandler = messageHandler
+    def __init__(self, mainWindow):
         self.mainWindow = mainWindow
         self.resolutions = {}
         # Macro dict item contents: tuple of (resolution, parameter values)
         self.macros = {}
 
-    def loadResolutionsFromDir(self, directory: Path) -> None:
+        self.loadResolutionsFromDir(
+            Path(self.mainWindow.SETTINGS.value("Program/BaseDir")) / "Core" / "Resolutions" / "Core")
+
+    def loadResolutionsFromDir(self, directory: Path) -> list:
+        resolutionsLoaded = []
         exceptionsCount = 0
         for resolution in listdir(directory):
             resolution = str(resolution)
@@ -52,19 +59,22 @@ class ResolutionManager:
                                                                            'category': resolutionCategory,
                                                                            'resolution': resClass
                                                                            }
-                    self.messageHandler.info(f"Loaded Resolution: {resNameString}")
+                    self.mainWindow.MESSAGEHANDLER.debug(f"Loaded Resolution: {resNameString}")
+                    resolutionsLoaded.append(f'{resolutionCategory}/{resNameString}')
             except Exception as e:
-                self.messageHandler.error(f"Cannot load resolutions from {str(directory)}" + "\n Info: " + repr(e))
+                self.mainWindow.MESSAGEHANDLER.error(
+                    f"Cannot load resolutions from {str(directory)}\n Info: {repr(e)}")
                 exceptionsCount += 1
                 if exceptionsCount > 3:
                     # Will not occur when loading modules with 3 or fewer resolutions, but that should be fine.
-                    self.messageHandler.critical("Failed loading too many resolutions to proceed.")
+                    self.mainWindow.MESSAGEHANDLER.critical("Failed loading too many resolutions to proceed.")
                     sys.exit(5)
+        return resolutionsLoaded
 
     def getResolutionParameters(self, resolutionCategory, resolutionNameString):
         resolutionsList = self.resolutions.get(resolutionCategory)
         if resolutionsList is not None and resolutionNameString in resolutionsList:
-            return self.resolutions[resolutionCategory][resolutionNameString]['parameters']
+            return dict(self.resolutions[resolutionCategory][resolutionNameString]['parameters'])
         return None
 
     def getResolutionOriginTypes(self, resolutionCategoryNameString: str) -> Union[list, None]:
@@ -136,8 +146,8 @@ class ResolutionManager:
             if resolutionName in self.resolutions.get(resolutionCategory):
                 if self.resolutions[resolutionCategory][resolutionName].get('resolution') == '':
                     # If resolution class does not exist locally, then assume it exists on the server.
-                    self.mainWindow.executeRemoteResolution(resolutionCategoryNameString, resolutionEntitiesInput,
-                                                            parameters, resolutionUID)
+                    self.mainWindow.FCOM.runRemoteResolution(
+                        resolutionCategoryNameString, resolutionEntitiesInput, parameters, resolutionUID)
                     # Returning a bool, so we know that the resolution is running on the server.
                     return True
                 resolutionClass = self.resolutions[resolutionCategory][resolutionName]['resolution']()
@@ -181,3 +191,513 @@ class ResolutionManager:
 
     def save(self) -> None:
         self.mainWindow.SETTINGS.setGlobalValue("Program/Macros", self.macros)
+
+
+class ResolutionParametersSelector(QtWidgets.QDialog):
+
+    def __init__(self, mainWindowObject, resolutionName, properties: dict, includeEntitySelector: list = None,
+                 originTypes: list = None, resolutionDescription: str = None,
+                 windowTitle: str = None) -> None:
+        super(ResolutionParametersSelector, self).__init__()
+
+        self.setModal(True)
+        if windowTitle is None:
+            windowTitle = f'Resolution Parameter Selector: {resolutionName}'
+        self.setWindowTitle(windowTitle)
+        self.parametersList = []
+        # Have two separate dicts for readability.
+        self.chosenParameters = {}
+        self.properties = properties
+        self.mainWindowObject = mainWindowObject
+        self.resolutionName = resolutionName
+
+        dialogLayout = QtWidgets.QGridLayout()
+        self.setLayout(dialogLayout)
+        self.childWidget = QtWidgets.QTabWidget()
+        dialogLayout.addWidget(self.childWidget, 0, 0, 4, 2)
+        dialogLayout.setRowStretch(0, 1)
+        dialogLayout.setColumnStretch(0, 1)
+
+        if includeEntitySelector is not None and originTypes is not None:
+            entitySelectTab = QtWidgets.QWidget()
+            entitySelectTab.setLayout(QtWidgets.QVBoxLayout())
+            labelText = ""
+            if resolutionDescription is not None:
+                labelText += resolutionDescription + "\n\n"
+            labelText += 'Select the entities to use for this resolution.\nAccepted Origin Types: ' + \
+                         ', '.join(originTypes)
+            entitySelectTabLabel = QtWidgets.QLabel(labelText)
+            entitySelectTabLabel.setWordWrap(True)
+            entitySelectTabLabel.setMaximumWidth(600)
+
+            entitySelectTabLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+            entitySelectTab.layout().addWidget(entitySelectTabLabel)
+
+            self.entitySelector = QtWidgets.QListWidget()
+            self.entitySelector.setSortingEnabled(True)
+            self.entitySelector.addItems(includeEntitySelector)
+
+            self.entitySelector.setSelectionMode(self.entitySelector.SelectionMode.MultiSelection)
+            entitySelectTab.layout().addWidget(self.entitySelector)
+
+            self.childWidget.addTab(entitySelectTab, 'Entities')
+
+        for key in properties:
+            propertyWidget = QtWidgets.QWidget()
+            propertyKeyLayout = QtWidgets.QVBoxLayout()
+            propertyWidget.setLayout(propertyKeyLayout)
+
+            propertyLabel = QtWidgets.QLabel(properties[key].get('description'))
+            propertyLabel.setWordWrap(True)
+            propertyLabel.setMaximumWidth(600)
+
+            propertyLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+            propertyKeyLayout.addWidget(propertyLabel)
+
+            propertyType = properties[key].get('type')
+            propertyValue = properties[key].get('value')
+            propertyDefaultValue = properties[key].get('default')
+
+            if propertyType == 'String':
+                propertyInputField = StringPropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'File':
+                propertyInputField = FilePropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'SingleChoice':
+                propertyInputField = SingleChoicePropertyInput(propertyValue, propertyDefaultValue)
+            elif propertyType == 'MultiChoice':
+                propertyInputField = MultiChoicePropertyInput(propertyValue, propertyDefaultValue)
+            else:
+                # If value has invalid type, skip to the next property.
+                propertyInputField = None
+
+            if propertyInputField is not None:
+                propertyKeyLayout.addWidget(propertyInputField)
+
+            rememberChoiceCheckbox = QtWidgets.QCheckBox('Remember Choice')
+            rememberChoiceCheckbox.setChecked(False)
+            propertyKeyLayout.addWidget(rememberChoiceCheckbox)
+            propertyKeyLayout.setStretch(1, 1)
+
+            self.childWidget.addTab(propertyWidget, key)
+            self.parametersList.append((key, propertyInputField, rememberChoiceCheckbox))
+
+        nextButton = QtWidgets.QPushButton('Next')
+        nextButton.clicked.connect(self.nextTab)
+        previousButton = QtWidgets.QPushButton('Previous')
+        previousButton.clicked.connect(self.previousTab)
+        acceptButton = QtWidgets.QPushButton('Accept')
+        acceptButton.setAutoDefault(True)
+        acceptButton.setDefault(True)
+        acceptButton.clicked.connect(self.accept)
+        cancelButton = QtWidgets.QPushButton('Cancel')
+        cancelButton.clicked.connect(self.reject)
+
+        dialogLayout.addWidget(previousButton, 4, 0, 1, 1)
+        dialogLayout.addWidget(nextButton, 4, 1, 1, 1)
+        dialogLayout.addWidget(cancelButton, 5, 0, 1, 1)
+        dialogLayout.addWidget(acceptButton, 5, 1, 1, 1)
+
+    def nextTab(self):
+        currentIndex = self.childWidget.currentIndex()
+        if currentIndex < self.childWidget.count():
+            self.childWidget.setCurrentIndex(currentIndex + 1)
+
+    def previousTab(self):
+        currentIndex = self.childWidget.currentIndex()
+        if currentIndex > 0:
+            self.childWidget.setCurrentIndex(currentIndex - 1)
+
+    def accept(self) -> None:
+        savedParameters = {}
+        for resolutionParameterName, resolutionParameterInput, resolutionParameterRemember in self.parametersList:
+            value = resolutionParameterInput.getValue()
+            if value == '':
+                msgBox = QtWidgets.QMessageBox()
+                msgBox.setModal(True)
+                QtWidgets.QMessageBox.warning(msgBox,
+                                              "Not all parameters were filled in",
+                                              "Some of the required parameters for the resolution have been left blank."
+                                              " Please fill them in before proceeding.")
+                return
+            self.chosenParameters[resolutionParameterName] = value
+
+            if resolutionParameterRemember.isChecked():
+                savedParameters[resolutionParameterName] = value
+
+        # Only save parameters after we verify that everything is filled in properly.
+        for savedParameter in savedParameters:
+            if self.properties[savedParameter].get('global') is True:
+                self.mainWindowObject.SETTINGS.setGlobalValue(
+                    f'Resolutions/Global/Parameters/{savedParameter}',
+                    savedParameters[savedParameter],
+                )
+            else:
+                self.mainWindowObject.SETTINGS.setGlobalValue(
+                    f'Resolutions/{self.resolutionName}/{savedParameter}',
+                    savedParameters[savedParameter],
+                )
+
+        super(ResolutionParametersSelector, self).accept()
+
+
+class ResolutionSearchResultsList(QtWidgets.QListWidget):
+
+    def __init__(self, mainWindowObject):
+        super(ResolutionSearchResultsList, self).__init__()
+        self.mainWindow = mainWindowObject
+        self.setSortingEnabled(True)
+
+    def mouseDoubleClickEvent(self, event: QtGui.QMouseEvent) -> None:
+        super(ResolutionSearchResultsList, self).mouseDoubleClickEvent(event)
+        resItem = self.itemAt(event.pos())
+        if resItem is None or '/' not in resItem.text():
+            return
+        self.mainWindow.centralWidget().tabbedPane.getCurrentScene().clearSelection()
+        self.mainWindow.runResolution(resItem.text())
+
+
+class FindResolutionDialog(QtWidgets.QDialog):
+
+    def __init__(self, parent, entityList: list, resolutionDict: dict):
+        super(FindResolutionDialog, self).__init__()
+        self.entities = entityList
+        self.resolutions = resolutionDict
+        self.setModal(True)
+        self.setWindowTitle('Find Resolutions')
+
+        dialogLayout = QtWidgets.QGridLayout()
+        self.setLayout(dialogLayout)
+
+        descriptionLabel = QtWidgets.QLabel("Find Resolutions based on their parameters.")
+        descriptionLabel.setWordWrap(True)
+        dialogLayout.addWidget(descriptionLabel, 0, 0, 1, 2)
+
+        originLabel = QtWidgets.QLabel("Origin Entity:")
+        self.originDropDown = QtWidgets.QComboBox()
+        self.originDropDown.addItem('Any')
+        self.originDropDown.addItems(entityList)
+        self.originDropDown.addItem('*')
+        dialogLayout.addWidget(originLabel, 1, 0, 1, 1)
+        dialogLayout.addWidget(self.originDropDown, 1, 1, 1, 1)
+
+        targetLabel = QtWidgets.QLabel("Target Entity:")
+        self.targetDropDown = QtWidgets.QComboBox()
+        self.targetDropDown.addItem('Any')
+        self.targetDropDown.addItems(entityList)
+        self.targetDropDown.addItem('*')
+        dialogLayout.addWidget(targetLabel, 2, 0, 1, 1)
+        dialogLayout.addWidget(self.targetDropDown, 2, 1, 1, 1)
+
+        keywordsLabel = QtWidgets.QLabel("Keywords:")
+        self.keywordsWidget = QtWidgets.QLineEdit()
+        self.keywordsWidget.setToolTip("Add keywords separated by spaces.\nKeywords are checked against the "
+                                       "resolutions' titles and descriptions.")
+        dialogLayout.addWidget(keywordsLabel, 3, 0, 1, 2)
+        dialogLayout.addWidget(self.keywordsWidget, 4, 0, 1, 2)
+
+        resultsLabel = QtWidgets.QLabel("Matches:")
+        self.resultsWidget = ResolutionSearchResultsList(parent)
+        self.resultsWidget.addItem('Click "Search" to display results')
+        dialogLayout.addWidget(resultsLabel, 5, 0, 1, 2)
+        dialogLayout.addWidget(self.resultsWidget, 6, 0, 2, 2)
+
+        self.searchButton = QtWidgets.QPushButton("Search")
+        self.searchButton.clicked.connect(self.search)
+        self.closeButton = QtWidgets.QPushButton("Close")
+        self.closeButton.clicked.connect(self.accept)
+        dialogLayout.addWidget(self.closeButton, 8, 0, 1, 1)
+        dialogLayout.addWidget(self.searchButton, 8, 1, 1, 1)
+
+    def search(self):
+        self.resultsWidget.clear()
+        target = self.targetDropDown.currentText()
+        validResolutions = []
+        for category in self.resolutions:
+            for resolution in self.resolutions[category]:
+                if target == 'Any':
+                    validResolutions.append(f'{category}/{resolution}')
+
+                elif target in self.resolutions[category][resolution]['originTypes']:
+                    validResolutions.append(f'{category}/{resolution}')
+        origin = self.originDropDown.currentText()
+        if origin != 'Any':
+            for category in self.resolutions:
+                for resolution in self.resolutions[category]:
+                    if origin not in self.resolutions[category][resolution]['originTypes']:
+                        with contextlib.suppress(KeyError):
+                            validResolutions.remove(f'{str(category)}/{str(resolution)}')
+        # Try to see if any of the keywords are a substring of the name or description of any resolution.
+        keywordFilter = self.keywordsWidget.text().strip()
+        if keywordFilter != '':
+            wordsToFind = keywordFilter.split(' ')
+            for category in self.resolutions:
+                for resolution in self.resolutions[category]:
+                    titleText = self.resolutions[category][resolution]['name']
+                    descriptionText = self.resolutions[category][resolution]['description']
+                    for keyword in wordsToFind:
+                        if keyword not in titleText and keyword not in descriptionText:
+                            with contextlib.suppress(KeyError):
+                                validResolutions.remove(f'{str(category)}/{str(resolution)}')
+        for result in validResolutions:
+            self.resultsWidget.addItem(result)
+
+
+class ResolutionExecutorThread(QtCore.QThread):
+    sig = QtCore.Signal(str, list, str)
+    sigStr = QtCore.Signal(str, str, str)
+    sigError = QtCore.Signal(str)
+
+    def __init__(self, resolution: str, resolutionArgument: list, resolutionParameters: dict,
+                 mainWindowObject, uid: str):
+        super().__init__()
+        self.resolution = resolution
+        self.resolutionArgument = resolutionArgument
+        self.resolutionParameters = resolutionParameters
+        self.mainWindow = mainWindowObject
+        self.return_results = True
+        self.uid = uid
+        self.done = False
+
+    def run(self) -> None:
+        try:
+            ret = self.mainWindow.RESOLUTIONMANAGER.executeResolution(self.resolution,
+                                                                      self.resolutionArgument,
+                                                                      self.resolutionParameters,
+                                                                      self.uid)
+            if ret is None:
+                self.sigError.emit(f'Resolution {self.resolution} failed during run.')
+            elif isinstance(ret, bool):
+                # Resolution is running on the server, we do not have results right now.
+                ret = None
+        except Exception as e:
+            self.sigError.emit(f'Resolution {self.resolution} failed during run: {str(e)}')
+            ret = None
+
+        # If the resolution is ran on the server or there is a problem, don't emit signal.
+        if ret is not None and self.return_results:
+            if isinstance(ret, str):
+                self.sigStr.emit(self.resolution, ret, self.uid)
+            else:
+                self.sig.emit(self.resolution, ret, self.uid)
+            self.done = True
+
+
+class MacroDialog(QtWidgets.QDialog):
+
+    def __init__(self, mainWindowObject):
+        super(MacroDialog, self).__init__()
+        self.mainWindowObject = mainWindowObject
+        self.setModal(True)
+
+        self.resolutionList = []
+        for category in mainWindowObject.RESOLUTIONMANAGER.getResolutionCategories():
+            self.resolutionList.extend(
+                f'{category}/{resolution}'
+                for resolution in mainWindowObject.RESOLUTIONMANAGER.getResolutionsInCategory(category)
+            )
+        self.resolutionList.sort()
+
+        layout = QtWidgets.QVBoxLayout()
+        self.setLayout(layout)
+
+        macroLabel = QtWidgets.QLabel("This is a list of all currently configured Macros.\n"
+                                      "Click on a Macro to view the Resolutions included in it.")
+        macroLabel.setWordWrap(True)
+        self.macroTree = MacroTree(self, mainWindowObject)
+        self.macroTree.setSelectionMode(self.macroTree.SelectionMode.ExtendedSelection)
+        self.macroTree.setSelectionBehavior(self.macroTree.SelectionBehavior.SelectRows)
+        self.macroTree.setHeaderLabels(['Macro UID', 'Delete'])
+        self.macroTree.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectionBehavior.SelectRows)
+        self.macroTree.setSortingEnabled(False)
+        # Stretch the first column, since it contains the primary field.
+        self.macroTree.header().setStretchLastSection(False)
+        self.macroTree.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeMode.Stretch)
+
+        buttonsWidget = QtWidgets.QWidget()
+        buttonsWidgetLayout = QtWidgets.QHBoxLayout()
+        buttonsWidget.setLayout(buttonsWidgetLayout)
+        closeButton = QtWidgets.QPushButton('Close')
+        closeButton.clicked.connect(self.reject)
+        addMacroButton = QtWidgets.QPushButton('Create New Macro')
+        addMacroButton.clicked.connect(self.createMacro)
+        runSelectedButton = QtWidgets.QPushButton('Run Selected Macros')
+        runSelectedButton.clicked.connect(self.accept)
+        buttonsWidgetLayout.addWidget(closeButton)
+        buttonsWidgetLayout.addWidget(addMacroButton)
+        buttonsWidgetLayout.addWidget(runSelectedButton)
+
+        layout.addWidget(macroLabel)
+        layout.addWidget(self.macroTree)
+        layout.addWidget(buttonsWidget)
+        self.updateMacroTree()
+        self.setBaseSize(1000, 1000)
+
+    def updateMacroTree(self) -> None:
+        self.macroTree.clear()
+        with self.mainWindowObject.macrosLock:
+            allMacros = self.mainWindowObject.RESOLUTIONMANAGER.macros
+
+            for macro in allMacros:
+                newMacro = MacroTreeItem(macro, allMacros[macro])
+                self.macroTree.addTopLevelItem(newMacro)
+                self.macroTree.setItemWidget(newMacro, 1, newMacro.deleteButton)
+
+    def createMacro(self) -> None:
+        createMacroDialog = MacroCreatorDialog(self.resolutionList)
+        if createMacroDialog.exec():
+            macroResolutionsList = []
+            numberOfResolutionsSelected = createMacroDialog.createList.count()
+            for itemIndex in range(numberOfResolutionsSelected):
+                itemText = createMacroDialog.createList.item(itemIndex).text()
+                resolutionCategory, resolutionName = itemText.split('/', 1)
+                rParameters = self.mainWindowObject.RESOLUTIONMANAGER.getResolutionParameters(resolutionCategory,
+                                                                                              resolutionName)
+                if rParameters is None:
+                    message = f'Resolution parameters not found for resolution: {resolutionName}'
+                    self.mainWindowObject.MESSAGEHANDLER.error(message, popUp=True, exc_info=False)
+                    self.mainWindowObject.setStatus(f'{message}, Macro creation aborted.')
+                    return
+
+                resolutionParameterValues = self.mainWindowObject.popParameterValuesAndReturnSpecified(resolutionName,
+                                                                                                       rParameters)
+
+                if rParameters:
+                    parameterSelector = ResolutionParametersSelector(
+                        self.mainWindowObject, resolutionName, rParameters,
+                        windowTitle=f'[{str(itemIndex + 1)}/{str(numberOfResolutionsSelected)}] Select Parameter '
+                                    f'values for Resolution: {resolutionName}')
+                    if parameterSelector.exec():
+                        resolutionParameterValues.update(parameterSelector.chosenParameters)
+                    else:
+                        self.mainWindowObject.MESSAGEHANDLER.info('Macro creation aborted.')
+                        self.mainWindowObject.setStatus('Macro creation aborted.')
+                        return
+
+                macroResolutionsList.append((itemText, resolutionParameterValues))
+            self.mainWindowObject.RESOLUTIONMANAGER.createMacro(macroResolutionsList)
+            self.updateMacroTree()
+            self.mainWindowObject.setStatus('New Macro Created.')
+            self.mainWindowObject.MESSAGEHANDLER.info('New Macro Created.')
+
+    def accept(self) -> None:
+        super(MacroDialog, self).accept()
+
+
+class MacroTree(QtWidgets.QTreeWidget):
+
+    def __init__(self, parent, mainWindowObject):
+        super(MacroTree, self).__init__(parent=parent)
+        self.mainWindowObject = mainWindowObject
+
+    def deleteMacro(self, treeEntry: QtWidgets.QTreeWidgetItem):
+        index = self.indexOfTopLevelItem(treeEntry)
+        uid = treeEntry.text(0)
+        self.mainWindowObject.RESOLUTIONMANAGER.deleteMacro(uid)
+        self.takeTopLevelItem(index)
+
+
+class MacroTreeItem(QtWidgets.QTreeWidgetItem):
+
+    def __init__(self, uid: str, resolutionList: list):
+        super(MacroTreeItem, self).__init__()
+        self.setText(0, uid)
+        self.uid = uid
+        self.setFlags(QtCore.Qt.ItemFlag.ItemIsEditable | self.flags())
+
+        self.deleteButton = QtWidgets.QPushButton('X')
+        self.deleteButton.clicked.connect(self.removeSelf)
+
+        for resolution in resolutionList:
+            resolutionItem = QtWidgets.QTreeWidgetItem()
+            resolutionItem.setText(0, f'Resolution: {resolution[0]}')
+            self.addChild(resolutionItem)
+
+    def setData(self, column: int, role: int, value: Any):
+        if self.treeWidget() is None:
+            # Happens during initialization
+            super().setData(column, role, value)
+
+        elif self.treeWidget().mainWindowObject.RESOLUTIONMANAGER.renameMacro(self.uid, value):
+            super().setData(column, role, value)
+            self.uid = value
+
+    def removeSelf(self):
+        self.treeWidget().deleteMacro(self)
+
+
+class MacroCreatorDialog(QtWidgets.QDialog):
+
+    def __init__(self, resolutionsWithCategoriesList: list):
+        super(MacroCreatorDialog, self).__init__()
+        self.setWindowTitle('Create new Macro')
+
+        self.viewList = QtWidgets.QListWidget()
+        for resolution in resolutionsWithCategoriesList:
+            viewItem = QtWidgets.QListWidgetItem(resolution)
+            self.viewList.addItem(viewItem)
+        self.viewList.sortItems()
+        self.viewList.setSelectionMode(self.viewList.SelectionMode.ExtendedSelection)
+
+        buttonsAddRemoveWidget = QtWidgets.QWidget()
+        buttonsAddRemoveWidgetLayout = QtWidgets.QVBoxLayout()
+        buttonAdd = QtWidgets.QPushButton('>')
+        buttonAdd.clicked.connect(self.addSelectedToMacro)
+        buttonRemove = QtWidgets.QPushButton('<')
+        buttonRemove.clicked.connect(self.removeSelectedFromMacro)
+        buttonsAddRemoveWidget.setLayout(buttonsAddRemoveWidgetLayout)
+        buttonsAddRemoveWidgetLayout.addWidget(buttonAdd)
+        buttonsAddRemoveWidgetLayout.addWidget(buttonRemove)
+        self.createList = QtWidgets.QListWidget()
+        buttonsRearrangeWidget = QtWidgets.QWidget()
+        buttonsRearrangeWidgetLayout = QtWidgets.QVBoxLayout()
+        buttonsRearrangeWidget.setLayout(buttonsRearrangeWidgetLayout)
+        buttonMoveUp = QtWidgets.QPushButton('^')
+        buttonMoveUp.clicked.connect(self.shiftSelectedUp)
+        buttonMoveDown = QtWidgets.QPushButton('v')
+        buttonMoveDown.clicked.connect(self.shiftSelectedDown)
+        buttonsRearrangeWidgetLayout.addWidget(buttonMoveUp)
+        buttonsRearrangeWidgetLayout.addWidget(buttonMoveDown)
+        allResolutionsLabel = QtWidgets.QLabel('All Resolutions')
+        allResolutionsLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        selectedResolutionsLabel = QtWidgets.QLabel('Selected Resolutions')
+        selectedResolutionsLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        confirmButton = QtWidgets.QPushButton('Confirm')
+        confirmButton.clicked.connect(self.accept)
+        cancelButton = QtWidgets.QPushButton('Cancel')
+        cancelButton.clicked.connect(self.reject)
+
+        macroCreateLayout = QtWidgets.QGridLayout()
+        self.setLayout(macroCreateLayout)
+        macroCreateLayout.addWidget(self.viewList, 1, 0, 1, 5)
+        macroCreateLayout.addWidget(buttonsAddRemoveWidget, 1, 5, 1, 1)
+        macroCreateLayout.addWidget(self.createList, 1, 6, 1, 5)
+        macroCreateLayout.addWidget(buttonsRearrangeWidget, 1, 11, 1, 1)
+        macroCreateLayout.addWidget(allResolutionsLabel, 0, 0, 1, 5)
+        macroCreateLayout.addWidget(selectedResolutionsLabel, 0, 6, 1, 5)
+        macroCreateLayout.addWidget(cancelButton, 2, 1, 1, 3)
+        macroCreateLayout.addWidget(confirmButton, 2, 7, 1, 3)
+        self.setBaseSize(1000, 700)
+
+    def addSelectedToMacro(self):
+        for selectedItem in self.viewList.selectedItems():
+            self.createList.addItem(QtWidgets.QListWidgetItem(selectedItem.text()))
+
+    def removeSelectedFromMacro(self):
+        for selectedItem in self.createList.selectedItems():
+            self.createList.takeItem(self.createList.row(selectedItem))
+
+    def shiftSelectedUp(self):
+        with contextlib.suppress(IndexError):
+            selectedItemIndex = self.createList.row(self.createList.selectedItems()[0])
+            if selectedItemIndex != 0:
+                currentItem = self.createList.takeItem(selectedItemIndex)
+                self.createList.insertItem(selectedItemIndex - 1, currentItem)
+                currentItem.setSelected(True)
+
+    def shiftSelectedDown(self):
+        with contextlib.suppress(IndexError):
+            selectedItemIndex = self.createList.row(self.createList.selectedItems()[0])
+            currentItem = self.createList.takeItem(selectedItemIndex)
+            self.createList.insertItem(selectedItemIndex + 1, currentItem)
+            currentItem.setSelected(True)
+

Core/Resolutions/Core/ASNToCIDR.py

@@ -10,7 +10,7 @@ class ASNToCIDR:
     originTypes = {'Autonomous System'}
 
     # A set of entities that could be the result of this resolution.
-    resultTypes = {'Network'}
+    resultTypes = {'Network', 'Company', 'Organization', 'Phrase'}
 
     # A dictionary of properties for this resolution. The key is the property name,
     # the value is the property attributes. The type of input expected from the user is determined by the

Core/Resolutions/Core/AffiliateCodesExtractor.py

@@ -44,10 +44,12 @@ class AffiliateCodesExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import urllib
         import tldextract
         import re
 
+        playwrightPath = Path(parameters['Playwright Firefox'])
         returnResults = []
         visitExternal = parameters['Visit External Links'] == 'Yes'
 
@@ -184,7 +186,7 @@ class AffiliateCodesExtractor:
                             GetAffiliateCodes(currentUID, newLink)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'

Core/Resolutions/Core/ContainsPhrase.py

@@ -22,7 +22,8 @@ class ContainsPhrase:
                                            'value': ''},
                   'Case Sensitive': {'description': 'Do you want the phrase to be case sensitive?',
                                      'type': 'SingleChoice',
-                                     'value': {'Yes', 'No'}
+                                     'value': {'Yes', 'No'},
+                                     'default': 'Yes'
                                      }
                   }
 

Core/Resolutions/Core/CryptoAddressExtractor.py

@@ -24,8 +24,10 @@ class CryptoAddressExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import re
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         returnResults = []
 
         ethRegex = re.compile(r"\b0[xX][a-fA-F0-9]{40}\b")
@@ -239,7 +241,7 @@ class CryptoAddressExtractor:
                                                         'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '

Core/Resolutions/Core/DecodeRedirectUrlParameter.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+
+class DecodeRedirectUrlParameter:
+    name = "Decode Redirect URL"
+    category = "Website Information"
+    description = "Search the URL's parameters to see where you'll be redirected."
+    originTypes = {'Website'}
+    resultTypes = {'Website'}
+
+    parameters = {}
+
+    def resolution(self, entityJsonList, parameters):
+        import contextlib
+        from urllib.parse import urlparse
+        from urllib.parse import parse_qs
+        from urllib.parse import unquote
+        from base64 import b64decode
+
+        def get_redirect_value(parameter_arg: str) -> str:
+            with contextlib.suppress(Exception):
+                clean_val = unquote(parameter_arg)
+                if urlparse(clean_val).scheme:
+                    return clean_val
+                clean_val = unquote(b64decode(parameter_arg).decode('UTF-8'))
+                if urlparse(clean_val).scheme:
+                    return clean_val
+            return ''
+
+        returnResults = []
+
+        for entity in entityJsonList:
+            primaryField = entity['URL'].strip()
+            parsed_url = urlparse(primaryField, allow_fragments=False)
+
+            parsed_url_params = parse_qs(parsed_url.query)
+            for param, param_value in parsed_url_params.items():
+                param_potential_url_value = get_redirect_value(', '.join(param_value))
+                if param_potential_url_value:
+                    parsed_url_params_copy = dict(parsed_url_params)
+                    parsed_url_params_copy.pop(param)
+                    new_entity = {'URL': param_potential_url_value,
+                                  'Entity Type': 'Website'}
+                    for param_copy, param_value_copy in parsed_url_params_copy.items():
+                        new_entity[param_copy] = ', '.join(param_value_copy)
+                    returnResults.append([new_entity,
+                                          {entity['uid']: {'Resolution': 'Redirect To',
+                                                           'Notes': ''}}])
+                    break
+
+        return returnResults

Core/Resolutions/Core/EmailExtractor.py

@@ -37,10 +37,12 @@ class EmailExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import re
         import contextlib
         from email_validator import validate_email, caching_resolver, EmailNotValidError
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         returnResults = []
 
         # Source: https://emailregex.com/
@@ -91,14 +93,17 @@ class EmailExtractor:
                 siteContent = re.sub(r'\s*(\[|\<|\()+\s*at\s*(\]|\>|\))+\s*', '@', siteContent)
                 siteContent = re.sub(r'\s*(\[|\<|\()+\s*dot\s*(\]|\>|\))+\s*', '.', siteContent)
                 siteContent = re.sub(r'\s*(\[|\<|\()+\s*\.\s*(\]|\>|\))+\s*', '.', siteContent)
+                siteContent = re.sub(r'\s*@\s*', '@', siteContent)
+                siteContent = re.sub(r'\s*\.', '.', siteContent)
+                siteContent = re.sub(r'\.\s*([^A-Z])', r'.\1', siteContent)
 
                 potentialEmails = emailRegex.findall(siteContent)
                 for potentialEmail in potentialEmails:
                     with contextlib.suppress(EmailNotValidError):
                         valid = validate_email(potentialEmail, dns_resolver=resolver, check_deliverability=verifyDomain)
-                        if valid.email not in allEmails:
-                            allEmails.add(valid.email)
-                            returnResults.append([{'Email Address': valid.email,
+                        if valid.normalized not in allEmails:
+                            allEmails.add(valid.normalized)
+                            returnResults.append([{'Email Address': valid.normalized,
                                                    'Entity Type': 'Email Address'},
                                                   {currentUID: {'Resolution': 'Email Address Found',
                                                                 'Notes': ''}}])
@@ -109,19 +114,17 @@ class EmailExtractor:
                     with contextlib.suppress(EmailNotValidError):
                         valid = validate_email(newLink[7:], dns_resolver=resolver,
                                                check_deliverability=verifyDomain)
-                        if valid.email not in allEmails:
-                            allEmails.add(valid.email)
-                            returnResults.append([{'Email Address': valid.email,
+                        if valid.normalized not in allEmails:
+                            allEmails.add(valid.normalized)
+                            returnResults.append([{'Email Address': valid.normalized,
                                                    'Entity Type': 'Email Address'},
                                                   {currentUID: {'Resolution': 'Email Address Found',
                                                                 'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
-                           'Chrome/101.0.4951.54 Safari/537.36'
+                viewport={'width': 1920, 'height': 1080}
             )
             for entity in entityJsonList:
                 uid = entity['uid']

Core/Resolutions/Core/ExtractPDFMeta.py

@@ -27,7 +27,7 @@ class ExtractPDFMeta:
             uid = entity['uid']
             filePath = Path(parameters['Project Files Directory']) / entity['File Path']
 
-            if not (filePath.exists() and filePath.is_file()):
+            if not (filePath.is_file()):
                 continue
 
             if magic.from_file(str(filePath), mime=True) != 'application/pdf':
@@ -39,10 +39,7 @@ class ExtractPDFMeta:
                 number_of_pages = len(pdf.pages)
 
             for metadataKey in info:
-                if metadataKey.startswith('/'):
-                    attrValue = metadataKey[1:]
-                else:
-                    attrValue = metadataKey
+                attrValue = metadataKey[1:] if metadataKey.startswith('/') else metadataKey
                 if 'Date' in metadataKey:
                     try:
                         strDate = info[metadataKey].split(':', 1)[1]
@@ -73,9 +70,7 @@ class ExtractPDFMeta:
                 else:
                     # Clean some misshapen strings
                     value = str(info[metadataKey])
-                    if value.startswith('/'):
-                        value = value[1:]
-
+                    value = value.removeprefix('/')
                     returnResults.append([{'Phrase': f'{attrValue}: {value}',
                                            'Entity Type': 'Phrase'},
                                           {uid: {'Resolution': attrValue, 'Notes': ''}}])

Core/Resolutions/Core/FileExtractor.py

@@ -45,11 +45,12 @@ class FileExtractor:
         import tldextract
         import requests
         from hashlib import md5
-        from binascii import hexlify
         from pathlib import Path
         from bs4 import BeautifulSoup
         from playwright.sync_api import sync_playwright, TimeoutError, Error
 
+        playwrightPath = Path(parameters['Playwright Firefox'])
+
         try:
             maxDepth = max(int(parameters['Max Depth']), 0)
         except ValueError:
@@ -117,7 +118,7 @@ class FileExtractor:
                                                          'Notes': ''}}])
 
                             docProperName = link.split('/')[-1]
-                            docFileName = f'{hexlify(md5(link.encode()).digest()).decode()} | {docProperName}'
+                            docFileName = f'{md5(link.encode("UTF-8")).hexdigest()}_{docProperName}'
                             docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
                             with contextlib.suppress(Exception):
@@ -156,7 +157,7 @@ class FileExtractor:
                                               {uid: {'Resolution': 'File URL',
                                                      'Notes': ''}}])
                         docProperName = link.split('/')[-1]
-                        docFileName = f'{hexlify(md5(link.encode()).digest()).decode()} | {docProperName}'
+                        docFileName = f'{md5(link.encode()).hexdigest()}_{docProperName}'
                         docFullPath = Path(parameters['Project Files Directory']) / docFileName
 
                         with contextlib.suppress(Exception):
@@ -180,7 +181,7 @@ class FileExtractor:
                     iterateOnDepth(newURL, newDepth)
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'

Core/Resolutions/Core/FileHasher.py

@@ -7,21 +7,21 @@ class FileHasher:
     description = "Get the Hash of a file."
     originTypes = {"Image", "Document", "Spreadsheet", "Video", "Archive", "Disk"}
     resultTypes = {'Hash'}
-    parameters = {'hashing_algorithms': {'description': 'The type of hash/es that will be returned',
-                                         'type': 'MultiChoice',
-                                         'value': {'SHA1', 'SHA256', 'MD5'}
-                                         }}
+    parameters = {'Hashing Algorithm': {'description': 'Choose the type of hash(es) that you want to be returned:',
+                                        'type': 'MultiChoice',
+                                        'value': {'SHA1', 'SHA256', 'MD5'}
+                                        }}
 
     def resolution(self, entityJsonList, parameters):
         import hashlib
         from pathlib import Path
 
         return_result = []
-        hashing_algorithms = parameters['hashing_algorithms']
+        hashing_algorithms = parameters['Hashing Algorithm']
         for entity in entityJsonList:
             uid = entity['uid']
             file_path = Path(parameters['Project Files Directory']) / entity['File Path']
-            if not (file_path.exists() and file_path.is_file()):
+            if not file_path.is_file():
                 continue
             block_size = 65536  # The size of each read from the file
             for hashing_algorithm in hashing_algorithms:

Core/Resolutions/Core/GetExternalURLs.py

@@ -39,8 +39,14 @@ class GetExternalURLs:
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
         import re
-        import urllib.parse
+        import contextlib
+        from urllib.parse import urlparse
+        from urllib.parse import parse_qs
+        from urllib.parse import unquote
+        from base64 import b64decode
+        from pathlib import Path
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         onionRegex = re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$""")
         returnResult = []
 
@@ -48,16 +54,23 @@ class GetExternalURLs:
         extract_img = '<img> elements' in parameters['Element types to check']
         extract_link = '<link> elements' in parameters['Element types to check']
 
-        # Sites like youtube replace external links with a redirect link originating
-        #   from the site itself. This sort of gets around that.
-        redirectRegex = re.compile(r'\?.*(q|url)=\S[^&#?]+', re.IGNORECASE)
+        def get_potential_redirect_value(potential_redirect_url: str) -> str:
+            parsed_url = urlparse(potential_redirect_url, allow_fragments=False)
+            parsed_url_params = parse_qs(parsed_url.query)
+            for param, param_value in parsed_url_params.items():
+                with contextlib.suppress(Exception):
+                    clean_val = unquote(', '.join(param_value))
+                    if urlparse(clean_val).scheme:
+                        return clean_val
+                    clean_val = unquote(b64decode(', '.join(param_value)).decode('UTF-8'))
+                    if urlparse(clean_val).scheme:
+                        return clean_val
+            return ''
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
-                           'Chrome/101.0.4951.64 Safari/537.36'
+                viewport={'width': 1920, 'height': 1080}
             )
             page = context.new_page()
             externalUrls = {}
@@ -65,7 +78,7 @@ class GetExternalURLs:
             for site in entityJsonList:
                 uid = site['uid']
                 url = site['URL']
-                parsedURL = urllib.parse.urlparse(url)
+                parsedURL = urlparse(url)
                 if not all([parsedURL.scheme, parsedURL.netloc]):
                     continue
                 domain = tldextract.extract(url).fqdn
@@ -75,31 +88,28 @@ class GetExternalURLs:
                 for _ in range(3):
                     try:
                         page.goto(url, wait_until="networkidle", timeout=10000)
+
+                        ### Youtube
+                        with contextlib.suppress(Exception):
+                            page.get_by_role("button",
+                                             name="Reject the use of cookies and other data for the purposes described").click()
+                        with contextlib.suppress(Exception):
+                            page.get_by_role("button", name="Show more").click()
                         soupContents = BeautifulSoup(page.content(), 'lxml')
 
                         if extract_a:
                             linksInAHref = soupContents.find_all('a')
                             for tag in linksInAHref:
                                 link = tag.get('href', None)
-                                parsedURL = urllib.parse.urlparse(link)
+                                parsedURL = urlparse(link)
                                 if all([parsedURL.scheme, parsedURL.netloc]):
                                     if domain in link:
-                                        redirectLinks = redirectRegex.findall(link)
-                                        if 'redirect' in link and len(redirectLinks) > 0:
+                                        redirectLink = get_potential_redirect_value(link)
+                                        if redirectLink:
                                             try:
-                                                newLink = str(urllib.parse.unquote(redirectLinks[0]))[2:]
-                                                try:
-                                                    externalUrls[newLink].add(uid)
-                                                except KeyError:
-                                                    externalUrls[newLink] = {uid}
-                                            except IndexError:
-                                                try:
-                                                    externalUrls[redirectLinks[0]].add(uid)
-                                                except KeyError:
-                                                    externalUrls[redirectLinks[0]] = {uid}
-                                            except Exception:
-                                                pass
-
+                                                externalUrls[redirectLink].add(uid)
+                                            except KeyError:
+                                                externalUrls[redirectLink] = {uid}
                                     else:
                                         newLink = link.split('#')[0].split('?')[0]
                                         try:
@@ -110,7 +120,7 @@ class GetExternalURLs:
                             linksInImgSrc = soupContents.find_all('img')
                             for tag in linksInImgSrc:
                                 link = tag.get('src', None)
-                                parsedURL = urllib.parse.urlparse(link)
+                                parsedURL = urlparse(link)
                                 if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
                                     newLink = link.split('#')[0].split('?')[0]
                                     try:
@@ -122,7 +132,7 @@ class GetExternalURLs:
                             linksInLinkHref = soupContents.find_all('link')
                             for tag in linksInLinkHref:
                                 link = tag.get('href', None)
-                                parsedURL = urllib.parse.urlparse(link)
+                                parsedURL = urlparse(link)
                                 if all([parsedURL.scheme, parsedURL.netloc]) and domain not in link:
                                     newLink = link.split('#')[0].split('?')[0]
                                     try:

Core/Resolutions/Core/GetInternalURLs.py

@@ -19,12 +19,14 @@ class GetInternalURLs:
         import tldextract
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import urllib.parse
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         returnResult = []
         internalUrls = {}
 
-        considerResources = False if parameters['Include Resources'] == 'Only consider links to pages' else True
+        considerResources = parameters['Include Resources'] != 'Only consider links to pages'
 
         def handleLink(currentLink, currentUrl, currentDomain):
             if currentLink is None:
@@ -35,17 +37,19 @@ class GetInternalURLs:
                 currentLink = currentUrl + currentLink[1:]
             elif currentLink.startswith('/'):
                 urlParts = urllib.parse.urlparse(currentUrl)
-                currentLink = urlParts.scheme + '://' + urlParts.netloc + currentLink
+                currentLink = f'{urlParts.scheme}://{urlParts.netloc}{currentLink}'
             parsedCurrentURL = urllib.parse.urlparse(currentLink)
-            if all([parsedCurrentURL.scheme, parsedCurrentURL.netloc]):
-                if currentDomain in currentLink:
-                    newLink = currentLink.split('#')[0].split('?')[0]
-                    if newLink.endswith('/'):
-                        newLink = newLink[:-1]
-                    return newLink
+            if (
+                all([parsedCurrentURL.scheme, parsedCurrentURL.netloc])
+                and currentDomain in currentLink
+            ):
+                newLink = currentLink.split('#')[0].split('?')[0]
+                if newLink.endswith('/'):
+                    newLink = newLink[:-1]
+                return newLink
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '

Core/Resolutions/Core/GetWebsiteText.py

@@ -21,8 +21,10 @@ class GetWebsiteText:
         from bs4.element import Comment
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from urllib.parse import urlparse
+        from pathlib import Path
 
         returnResults = []
+        playwrightPath = Path(parameters['Playwright Chromium'])
 
         def tag_visible(element):
             if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
@@ -36,7 +38,7 @@ class GetWebsiteText:
             return u" ".join(t.strip() for t in visible_texts if t.strip() != '')
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '

Core/Resolutions/Core/HostnameToDomain.py

@@ -16,8 +16,7 @@ class HostnameToDomain:
         for entity in entityJsonList:
             uid = entity['uid']
             primary_field = entity[list(entity)[1]].strip()
-            tsd, td, tsu = extract(primary_field)
-            domain = f'{td}.{tsu}'
+            domain = extract(primary_field).fqdn
             if domain == primary_field:
                 continue
             return_result.append([{'Domain Name': domain,

Core/Resolutions/Core/JSCodeExtractor.py

@@ -19,9 +19,11 @@ class JSCodeExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, Error
         from base64 import b64decode
+        from pathlib import Path
         import re
         import contextlib
 
+        playwrightPath = Path(parameters['Playwright Firefox'])
         returnResults = []
         requestUrlsParsed = set()
 
@@ -29,7 +31,7 @@ class JSCodeExtractor:
         pubRegex = re.compile(r'\bca-pub-\d{1,16}\b', re.IGNORECASE)
         gtmRegex = re.compile(r'\bGTM-[A-Z\d]{1,7}\b')
         gRegex = re.compile(r'\bG-[A-Z\d]{1,15}\b', re.IGNORECASE)
-        qualtricsRegex = re.compile(r'\bQ_(?:Z|S)ID=\w*\b')
+        qualtricsRegex = re.compile(r'\bQ_[ZS]ID=\w*\b')
         pingdomRegex = re.compile(r'\bpa-[a-fA-F\d]{24}.js\b$')
         mPulseRegex = re.compile(r'go-mpulse.net/boomerang/[A-Z\d]{5}(?:-[A-Z\d]{5}){4}\b')
         contextWebRegex = re.compile(r'\.contextweb\.com.*token=.*')
@@ -173,7 +175,7 @@ class JSCodeExtractor:
                                                  'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.firefox.launch()
+            browser = p.firefox.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'

Core/Resolutions/Core/LongANStringExtractor.py

@@ -29,8 +29,10 @@ class LongANStringExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import re
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
         try:
             minLength = int(parameters['Minimum Length'])
             if minLength < 1:
@@ -88,7 +90,7 @@ class LongANStringExtractor:
                                                         'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '

Core/Resolutions/Core/NPMJSSearch.py

@@ -21,8 +21,7 @@ class NPMJSSearch:
         for entity in entityJsonList:
             primaryField = entity[list(entity)[1]].lower()
 
-            string_checks = set()
-            string_checks.add(''.join(primaryField.split(' ')))
+            string_checks = {''.join(primaryField.split(' '))}
             string_checks.add('_'.join(primaryField.split(' ')))
             string_checks.add('-'.join(primaryField.split(' ')))
 

Core/Resolutions/Core/PhoneNumbersExtractor.py

@@ -24,8 +24,11 @@ class PhoneNumbersExtractor:
     def resolution(self, entityJsonList, parameters):
         from playwright.sync_api import sync_playwright, TimeoutError, Error
         from bs4 import BeautifulSoup
+        from pathlib import Path
         import re
 
+        playwrightPath = Path(parameters['Playwright Chromium'])
+
         cleanTagsRegex = re.compile(r'<.*?>')
         phoneNumCharsExclusion = re.compile(r'[^ -+()\[\]\d]')
 
@@ -74,7 +77,7 @@ class PhoneNumbersExtractor:
                                                             'Notes': ''}}])
 
         with sync_playwright() as p:
-            browser = p.chromium.launch()
+            browser = p.chromium.launch(executable_path=playwrightPath)
             context = browser.new_context(
                 viewport={'width': 1920, 'height': 1080},
                 user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '

Core/Resolutions/Core/WebsiteFromPhrase.py

@@ -15,7 +15,7 @@ class WebsiteFromPhrase:
         import re
         import tldextract
 
-        websiteRegex = re.compile(r"""^https?://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$""")
+        websiteRegex = re.compile(r"""^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$""")
         wordChar = re.compile(r'\w')
 
         returnResults = []

Core/Resolutions/Core/WordCounter.py

@@ -9,7 +9,8 @@ class WordCounter:
     resultTypes = {'Phrase'}
     parameters = {'Primary field or Notes': {'description': 'Choose Either Primary field or Notes',
                                              'type': 'SingleChoice',
-                                             'value': {'Notes', 'Primary Field'}}}
+                                             'value': {'Notes', 'Primary Field'},
+                                             'default': 'Notes'}}
 
     def resolution(self, entityJsonList, parameters):
         import re

Core/ResourceHandler.py

@@ -3,7 +3,7 @@
 
 import contextlib
 import re
-from typing import Union
+from typing import Union, Optional
 from glob import glob
 
 import networkx as nx
@@ -17,21 +17,55 @@ from ast import literal_eval
 from base64 import b64decode
 from dateutil import parser
 
-from PySide6.QtCore import QByteArray, QSize, QUrl, Qt
+from PIL import Image
+from PIL.ImageQt import ImageQt
+from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize, QUrl, Qt
 from PySide6 import QtWidgets, QtGui
 
 
+def resizePictureFromBuffer(picBuffer: QByteArray, newSize: tuple) -> QByteArray:
+    """
+    newSize: First is width, second is height.
+    """
+    picBufferData = picBuffer.data()
+    if picBufferData.startswith(b'<svg ') or picBufferData.startswith(b'<?xml '):
+        return QByteArray(resizeSVG(picBufferData, newSize))
+    originalImage = QtGui.QImage()
+    originalImage.loadFromData(picBuffer)
+    newImage = originalImage.scaled(newSize[0], newSize[1])
+
+    pictureByteArray = QByteArray()
+    imageBuffer = QBuffer(pictureByteArray)
+    imageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+    newImage.save(imageBuffer, "PNG")
+    imageBuffer.close()
+
+    return pictureByteArray
+
+
+def resizeSVG(byteString: bytes, resize: tuple):
+    bytesWidth = str(resize[0]).encode('UTF-8')
+    bytesHeight = str(resize[1]).encode('UTF-8')
+    widthRegex = re.compile(b' width="\d*" ')
+    for widthMatches in widthRegex.findall(byteString):
+        byteString = byteString.replace(widthMatches, b' ')
+    heightRegex = re.compile(b' height="\d*" ')
+    for heightMatches in heightRegex.findall(byteString):
+        byteString = byteString.replace(heightMatches, b' ')
+    return byteString.replace(b'<svg ', b'<svg height="%b" width="%b" ' % (bytesHeight, bytesWidth), 1)
+
+
 class ResourceHandler:
 
     def getIcon(self, iconName: str):
         return self.icons[iconName]
 
     # Load all resources needed.
-    def __init__(self, mainWindow, messageHandler) -> None:
+    def __init__(self, mainWindow) -> None:
         self.mainWindow = mainWindow
-        self.messageHandler = messageHandler
         self.programBaseDirPath = Path(self.mainWindow.SETTINGS.value("Program/BaseDir"))
         self.entityCategoryList = {}
+        self.moduleAssetPaths = []
 
         self.icons = {"uploading": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploading.png"),
                       "uploaded": str(self.programBaseDirPath / "Resources" / "Icons" / "Uploaded.png"),
@@ -61,7 +95,7 @@ class ResourceHandler:
             r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)])"""),
             'Phonenumber': re.compile(r"""^(\+|00)?[0-9() \-]{3,32}$"""),
             'String': re.compile(r""".+"""),
-            'URL': re.compile(r"""^(?:(?:http|ftp)s?|file)://(\S(?<!\.)){1,63}(\.(\S(?<!\.)){1,63})+$"""),
+            'URL': re.compile(r"""[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)"""),
             'Onion': re.compile(r"""^https?://\w{56}\.onion/?(\S(?<!\.))*(\.(\S(?<!\.))*)?$"""),
             'Domain': re.compile(r"""^(\S(?<!\.)(?!/)(?<!/)){1,63}(\.(\S(?<!\.)(?!/)(?<!/)){1,63})+$"""),
             'Float': re.compile(r"""^([-+])?(\d|\.(?=\d))+$"""),
@@ -78,7 +112,44 @@ class ResourceHandler:
             'ISINID': re.compile(r"""^[a-zA-Z0-9]{2}-?[a-zA-Z0-9]{9}-?[a-zA-Z0-9]$"""),
             'SIC/NAICS': re.compile(r"""^[0-9]{4,6}$""")}
 
-        self.loadCoreEntities()
+        self.loadModuleEntities(self.programBaseDirPath / "Core")
+
+    def getPictureFromFile(self, filePath: Path, resize: tuple = (0, 0)) -> Optional[QByteArray]:
+        """
+        resize: tuple, first is width, second is height.
+        """
+        try:
+            with open(filePath, 'rb') as newIconFile:
+                fileContents = newIconFile.read()
+            if fileContents.startswith(b'<svg ') or fileContents.startswith(b'<?xml '):
+                if resize != (0, 0):
+                    fileContents = resizeSVG(fileContents, resize)
+                pictureByteArray = QByteArray(fileContents)
+            else:
+                image = Image.open(str(filePath))
+                if resize != (0, 0):
+                    thumbnail = ImageQt(image.resize(resize))
+                else:
+                    thumbnail = ImageQt(image)
+                pictureByteArray = QByteArray()
+                imageBuffer = QBuffer(pictureByteArray)
+
+                imageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
+
+                thumbnail.save(imageBuffer, "PNG")
+                imageBuffer.close()
+        except ValueError as ve:
+            # Image type is unsupported (for ImageQt)
+            # Supported types: 1, L, P, RGB, RGBA
+            self.mainWindow.MESSAGEHANDLER.warning(f'Invalid Image selected: {str(ve)}', popUp=True)
+            pictureByteArray = None
+
+        return pictureByteArray
+
+    def loadModuleBanners(self, modulePath: Path):
+        assetsPath = modulePath / "Assets"
+        self.banners.update({f"{bannerPath.split('Banner_')[-1].split('.')[0]}": str(bannerPath)
+                             for bannerPath in glob(str(assetsPath / "Banner_*.svg"))})
 
     def getEntityCategories(self) -> list:
         return list(self.entityCategoryList)
@@ -88,8 +159,7 @@ class ResourceHandler:
         for entity in self.entityCategoryList[category]:
             entityValue = self.entityCategoryList[category][entity]
             eList.append((self.getBareBonesEntityJson(entity),
-                          entityValue['Icon']
-                          ))
+                          entityValue['Icon']))
         return eList
 
     def getEntityAttributes(self, entityType) -> Union[None, list]:
@@ -100,8 +170,8 @@ class ResourceHandler:
                     aList.extend(iter(self.entityCategoryList[category][entityType]['Attributes']))
                     break
         except KeyError:
-            self.messageHandler.error("Attempted to get attributes for "
-                                      "nonexistent entity type: " + str(entityType), True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for nonexistent entity type: {entityType}", True)
             return None
         return aList
 
@@ -157,13 +227,24 @@ class ResourceHandler:
         result = attrCheck.findall(attribute)
         return len(result) == 1
 
-    def addRecognisedEntityTypes(self, entityFile: Path) -> bool:
+    def getIconPathForIconFile(self, iconFile: str) -> Union[None, Path]:
+        iconPath = self.programBaseDirPath / "Resources" / "Icons" / iconFile
+        if iconPath.exists():
+            return iconPath
+        for assetPath in self.moduleAssetPaths:
+            iconPath = assetPath / iconFile
+            if iconPath.exists():
+                return iconPath
+        return self.programBaseDirPath / "Resources" / "Icons" / "Default.svg"
+
+    def addRecognisedEntityTypes(self, entityFile: Path) -> list:
+        entityTypesAdded = []
         try:
             tree = parse(entityFile, forbid_dtd=True, forbid_entities=True, forbid_external=True)
         except Exception as exc:
-            self.mainWindow.MESSAGEHANDLER.warning('Error occurred when loading entities from '
-                                                   + str(entityFile) + ': ' + str(exc) + ', skipping.')
-            return False
+            self.mainWindow.MESSAGEHANDLER.warning(
+                f'Error occurred when loading entities from {entityFile}: {exc}, skipping.')
+            return []
 
         root = tree.getroot()
 
@@ -198,26 +279,28 @@ class ResourceHandler:
                     self.entityCategoryList[category] = {}
                 self.entityCategoryList[category][entityName] = {
                     'Attributes': attributesDict,
-                    'Icon': str(self.programBaseDirPath / "Resources" / "Icons" / icon)}
+                    'Icon': str(self.getIconPathForIconFile(icon))}
+                entityTypesAdded.append(f'{category}/{entityName}')
             except (KeyError, AttributeError) as err:
                 # Ignore malformed entities
-                self.messageHandler.error(f'Error: {str(err)}', popUp=False)
+                self.mainWindow.MESSAGEHANDLER.error(f'Error: {str(err)}', popUp=False)
                 continue
-        return True
+        return entityTypesAdded
 
-    def loadCoreEntities(self) -> None:
-        entDir = self.programBaseDirPath / "Core" / "Entities"
-        for entFile in listdir(entDir):
-            if entFile.endswith('.xml'):
-                self.addRecognisedEntityTypes(entDir / entFile)
-
-    def loadModuleEntities(self) -> None:
-        entDir = self.programBaseDirPath / "Modules"
-        for module in listdir(entDir):
-            for entFile in listdir(entDir / module):
+    def loadModuleEntities(self, modulePath: Path) -> list:
+        entitiesPath = modulePath / 'Entities'
+        allModuleEntitiesAdded = []
+        if entitiesPath.exists():
+            for entFile in listdir(entitiesPath):
                 if entFile.endswith('.xml'):
-                    self.addRecognisedEntityTypes(
-                        entDir / module / entFile)
+                    allModuleEntitiesAdded += self.addRecognisedEntityTypes(entitiesPath / entFile)
+        return allModuleEntitiesAdded
+
+    def loadModuleAssets(self, modulePath: Path):
+        moduleAssetsPath = modulePath / "Assets"
+        if moduleAssetsPath.exists():
+            self.moduleAssetPaths.append(moduleAssetsPath)
+            self.loadModuleBanners(modulePath)
 
     def getEntityJson(self, entityType: str, jsonData=None) -> Union[dict, None]:
         eJson = {'uid': str(uuid4())}
@@ -230,7 +313,8 @@ class ResourceHandler:
                         eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
                     break
         except KeyError:
-            self.messageHandler.error(f"Attempted to get attributes for malformed entity type: {entityType}", True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for malformed entity type: {entityType}", True)
             return None
         eJson['Entity Type'] = entityType
         eJson['Date Created'] = None
@@ -267,8 +351,8 @@ class ResourceHandler:
                         if self.entityCategoryList[category][entityType]['Attributes'][attribute][2]:
                             return attribute
         except KeyError:
-            self.messageHandler.error(f"Attempted to get primary attribute for malformed entity type: {entityType}",
-                                      True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get primary attribute for malformed entity type: {entityType}", True)
         return None
 
     def getBareBonesEntityJson(self, entityType: str) -> Union[dict, None]:
@@ -280,7 +364,8 @@ class ResourceHandler:
                         eJson[attribute] = self.entityCategoryList[category][entityType]['Attributes'][attribute][0]
                     break
         except KeyError:
-            self.messageHandler.error(f"Attempted to get attributes for malformed entity type: {entityType}", True)
+            self.mainWindow.MESSAGEHANDLER.error(
+                f"Attempted to get attributes for malformed entity type: {entityType}", True)
             return None
         eJson['Entity Type'] = entityType
 
@@ -314,7 +399,7 @@ class ResourceHandler:
 
         return linkJson
 
-    def getEntityDefaultPicture(self, entityType) -> QByteArray:
+    def getEntityDefaultPicture(self, entityType: str) -> QByteArray:
         picture = self.programBaseDirPath / "Resources" / "Icons" / "Default.svg"
         try:
             for category in self.entityCategoryList:
@@ -324,8 +409,8 @@ class ResourceHandler:
                         picture = entityPicture
                     break
         except KeyError:
-            self.messageHandler.warning("Attempted to get icon for "
-                                        "nonexistent entity type: " + str(entityType), popUp=False)
+            self.mainWindow.MESSAGEHANDLER.warning(
+                f"Attempted to get icon for nonexistent entity type: {entityType}", popUp=False)
         finally:
             with open(picture, 'rb') as pictureFile:
                 pictureContents = pictureFile.read()

Core/SettingsObject.py

@@ -23,11 +23,17 @@ class SettingsObject(dict):
     def __init__(self):
         super().__init__()
         self.globalSettings = QSettings()
-        self.globalSettings.setValue("Program/Version", "v1.5.0")
+        self.globalSettings.setValue("Program/Version", "v1.6.5")
         self.globalSettings.setValue("Program/TOR Profile Location",
                                      self.globalSettings.value("Program/TOR Profile Location", ""))
         self.globalSettings.setValue("Program/BaseDir",
                                      self.globalSettings.value("Program/BaseDir", "Unset"))
+        self.globalSettings.setValue("Program/Version Check Source",
+                                     self.globalSettings.value("Program/Version Check Source",
+                                                               "https://api.github.com/repos/AccentuSoft/LinkScope_Client/releases/latest"))
+        self.globalSettings.setValue("Program/Update Source",
+                                     self.globalSettings.value("Program/Update Source",
+                                                               "https://github.com/AccentuSoft/LinkScope_Client/releases/latest/download/"))
 
         # The value '20' equates to logging.INFO
         # It's not necessary to set this, but we will for
@@ -59,6 +65,19 @@ class SettingsObject(dict):
         self.globalSettings.setValue("Program/Graphics/Label Fade Scroll Distance",
                                      self.globalSettings.value("Program/Graphics/Label Fade Scroll Distance", "3"))
 
+        self.globalSettings.setValue("Program/Usage/First Time Start",
+                                     self.globalSettings.value("Program/Usage/First Time Start", 'true'))
+
+        self.globalSettings.setValue("Program/Sources/Sources List",
+                                     self.globalSettings.value(
+                                         "Program/Sources/Sources List",
+                                         {}))
+
+        self.globalSettings.setValue("Program/Sources/Module Packs List",
+                                     self.globalSettings.value(
+                                         "Program/Sources/Module Packs List",
+                                         {}))
+
         self.setValue("Project/Name", "Untitled")
         self.setValue("Project/BaseDir", "")
         self.setValue("Project/FilesDir", "")
@@ -76,10 +95,11 @@ class SettingsObject(dict):
     def getGroupSettings(self, settingsGroup: str) -> dict:
         if not settingsGroup.endswith('/'):
             settingsGroup += '/'
-        settingsDict = {}
-        for setting in self.globalSettings.allKeys():
-            if setting.startswith(settingsGroup):
-                settingsDict[setting] = self.globalSettings.value(setting)
+        settingsDict = {
+            setting: self.globalSettings.value(setting)
+            for setting in self.globalSettings.allKeys()
+            if setting.startswith(settingsGroup)
+        }
         for setting in self:
             if setting.startswith(settingsGroup):
                 settingsDict[setting] = self[setting]
@@ -122,7 +142,7 @@ class SettingsObject(dict):
         self.globalSettings.sync()
         globalSettingsSavingError = self.globalSettings.status()
         if globalSettingsSavingError != self.globalSettings.Status.NoError:
-            raise Exception(f'Could not save global settings: {globalSettingsSavingError}')
+            raise ValueError(f'Could not save global settings: {globalSettingsSavingError}')
 
     def load(self, savedDict: dict) -> None:
         # No need to do anything with global settings.

Core/URLManager.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from os import symlink
 from shutil import copy2
 from hashlib import sha3_512
-from binascii import hexlify
 from urllib.parse import urlparse
 
 from PySide6 import QtCore
@@ -58,24 +57,22 @@ class URLManager:
         fileType = magic.from_file(urlPathString, mime=True)
         fileTypeSplit1, fileTypeSplit2 = fileType.split('/', 1)
         # CSV files not considered - may have any dialect, hard to accommodate.
-        if urlPath.suffix in ('.ods', '.xls', '.xlsm', '.xlsx') and \
-                fileTypeSplit2 in ('vnd.oasis.opendocument.spreadsheet',
+        if urlPath.suffix in {'.ods', '.xls', '.xlsm', '.xlsx'} and \
+                fileTypeSplit2 in {'vnd.oasis.opendocument.spreadsheet',
                                    'vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-                                   'vnd.ms-excel',
-                                   'vnd.openxmlformats-officedocument.spreadsheetml.sheet'):
-            entityJson = {"Spreadsheet Name": urlName,
-                          "File Path": savePathString,
-                          "Entity Type": "Spreadsheet"}
+                                   'vnd.ms-excel'}:
+            return {"Spreadsheet Name": urlName,
+                    "File Path": savePathString,
+                    "Entity Type": "Spreadsheet"}
         # Only support zip files for archives (for now) 10/Jul/2021).
         elif zipfile.is_zipfile(urlPathString):
-            entityJson = {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
+            return {"Archive Name": urlName, "File Path": savePathString, "Entity Type": "Archive"}
         elif fileTypeSplit1 == "video":
-            entityJson = {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
+            return {"Video Name": urlName, "File Path": savePathString, "Entity Type": "Video"}
         elif fileTypeSplit1 == "image":
-            entityJson = {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
+            return {"Image Name": urlName, "File Path": savePathString, "Entity Type": "Image"}
         else:
-            entityJson = {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
-        return entityJson
+            return {"Document Name": urlName, "File Path": savePathString, "Entity Type": "Document"}
 
     def moveURLToProjectFilesHelperIfNeeded(self, urlPath: Path):
         valuePath = Path(urlPath).absolute()
@@ -92,8 +89,8 @@ class URLManager:
 
             projectFilesPath = Path(self.mainWindow.SETTINGS.value("Project/FilesDir"))
             # Create a unique path in Project Files
-            saveHash = hexlify(sha3_512(str(urlPath).encode()).digest()).decode()[:16]  # nosec
-            savePath = projectFilesPath / f'{saveHash}|{urlPath.name}'
+            saveHash = sha3_512(str(urlPath).encode()).hexdigest()[:16]  # nosec
+            savePath = projectFilesPath / f'{saveHash}_{urlPath.name}'
 
             if createSymlink:
                 symlink(urlPath, savePath)

Core/UpdateManager.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+
+import contextlib
+import platform
+import subprocess
+import requests
+import tempfile
+import shutil
+import os
+import ctypes
+
+from pathlib import Path
+from semver import compare
+from PySide6 import QtCore, QtWidgets
+
+
+class UpdateManager:
+
+    def __init__(self, mainWindow):
+        self.updateThread = None
+        self.mainWindow = mainWindow
+        self.system = platform.system()
+        if self.system == 'Windows':
+            self.baseSoftwarePath = Path(os.path.abspath(os.sep)) / 'Program Files' / 'LinkScope'
+        else:
+            self.baseSoftwarePath = Path(os.path.abspath(os.sep)) / 'usr' / 'local' / 'sbin' / 'LinkScope'
+
+    def getDownloadURL(self):
+        downloadURLBase = self.mainWindow.SETTINGS.value("Program/Update Source")
+        if self.system == 'Windows':
+            return f"{downloadURLBase}LinkScope-Windows-x64.7z"
+        else:
+            return f"{downloadURLBase}LinkScope-Ubuntu-x64.7z"
+
+    def getLatestVersion(self):
+        with contextlib.suppress(Exception):
+            version_req = requests.get(self.mainWindow.SETTINGS.value("Program/Version Check Source"),
+                                       headers={'User-Agent': 'LinkScope Update Checker'})
+            if version_req.status_code == 200:
+                return version_req.json()['tag_name']
+        return None
+
+    def isUpdateAvailable(self):
+        latest_version = self.getLatestVersion()
+        return (
+                latest_version is not None
+                and compare(self.mainWindow.SETTINGS.value("Program/Version").lstrip('v'),
+                            latest_version.lstrip('v')) < 0
+        )
+
+    def doUpdate(self) -> None:
+        if self.updateThread is not None:
+            self.mainWindow.MESSAGEHANDLER.error('Update already in progress.')
+            return
+        self.mainWindow.MESSAGEHANDLER.info('Starting update...')
+        self.updateThread = UpdaterThread(self, self.mainWindow)
+        self.updateThread.updateDoneSignal.connect(self.finalizeUpdate)
+        self.updateThread.start()
+        self.mainWindow.MESSAGEHANDLER.info('Update in progress')
+
+    def finalizeUpdate(self, updateTempPath: str) -> None:
+        if updateTempPath != '':
+            latest_version = self.getLatestVersion()
+            self.mainWindow.SETTINGS.setValue("Program/Version", latest_version)
+            self.mainWindow.MESSAGEHANDLER.info('Updating done, please restart for the changes to take effect.')
+            self.mainWindow.MESSAGEHANDLER.info("The application will now save and close to apply the updates. "
+                                                "Please wait for a few minutes before reopening LinkScope.",
+                                                popUp=True)
+
+            uncompressNewVersion(
+                self.system,
+                self.mainWindow.SETTINGS.value("Program/BaseDir"),
+                str(self.baseSoftwarePath),
+                updateTempPath)
+            self.mainWindow.close()
+
+        else:
+            self.mainWindow.MESSAGEHANDLER.info('Updating failed.')
+
+
+class UpdaterWindow(QtWidgets.QDialog):
+
+    def __init__(self, mainWindow, updateManager: UpdateManager, updateAvailableOverride: bool = False):
+        super().__init__()
+        self.mainWindow = mainWindow
+        self.updateManager = updateManager
+
+        self.setWindowTitle('Update Manager')
+
+        layout = QtWidgets.QGridLayout()
+        self.setLayout(layout)
+
+        doUpdateButton = QtWidgets.QPushButton('Update')
+        doUpdateButton.clicked.connect(self.initiateUpdate)
+        cancelButton = QtWidgets.QPushButton('Close')
+        cancelButton.clicked.connect(self.reject)
+
+        updateAvailableLabel = QtWidgets.QLabel()
+        updateAvailableLabel.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
+        if updateAvailableOverride or updateManager.isUpdateAvailable():
+            updateAvailableLabel.setText('An update for LinkScope is available.')
+        else:
+            updateAvailableLabel.setText('LinkScope is up to date.')
+            doUpdateButton.setDisabled(True)
+            doUpdateButton.setEnabled(False)
+
+        layout.addWidget(updateAvailableLabel, 1, 1, 1, 2)
+        layout.addWidget(cancelButton, 2, 1)
+        layout.addWidget(doUpdateButton, 2, 2)
+
+    def initiateUpdate(self) -> None:
+        self.updateManager.doUpdate()
+        self.accept()
+
+
+class UpdaterThread(QtCore.QThread):
+    updateDoneSignal = QtCore.Signal(str)
+
+    def __init__(self, updateManager, mainWindow):
+        super().__init__()
+        self.mainWindow = mainWindow
+        self.updateManager = updateManager
+
+    def run(self) -> None:
+        try:
+            downloadUrl = self.updateManager.getDownloadURL()
+            clientTempCompressedArchive = tempfile.mkstemp(suffix='.7z')
+            tempPath = clientTempCompressedArchive[1]
+
+            with os.fdopen(clientTempCompressedArchive[0], 'wb') as tempArchive:
+                with requests.get(downloadUrl, stream=True) as fileStream:
+                    for chunk in fileStream.iter_content(chunk_size=5 * 1024 * 1024):
+                        tempArchive.write(chunk)
+            self.updateDoneSignal.emit(tempPath)
+        except Exception:
+            self.updateDoneSignal.emit('')
+
+
+def uncompressNewVersion(system: str, baseDir: str, baseSoftwarePath: str, updateTempPath: str):
+    tempDir = tempfile.mkdtemp(prefix='LinkScope_Updater_TMP_')
+
+    if system == 'Windows':
+        updaterPath = Path(baseDir) / "UpdaterUtil.exe"
+
+        tempUpdaterPath = Path(tempDir) / updaterPath.name
+        shutil.copy(updaterPath, tempUpdaterPath)
+
+        # This is done so that Windows spawns the updater as a detached process.
+        ShellExecuteEx = ctypes.windll.shell32.ShellExecuteEx
+        SEE_MASK_NO_CONSOLE = 0x00008000
+
+        class SHELLEXECUTEINFO(ctypes.Structure):
+            _fields_ = [
+                ("cbSize", ctypes.c_ulong),
+                ("fMask", ctypes.c_ulong),
+                ("hwnd", ctypes.c_void_p),
+                ("lpVerb", ctypes.c_char_p),
+                ("lpFile", ctypes.c_char_p),
+                ("lpParameters", ctypes.c_char_p),
+                ("lpDirectory", ctypes.c_char_p),
+                ("nShow", ctypes.c_int),
+                ("hInstApp", ctypes.c_void_p),
+                ("lpIDList", ctypes.c_void_p),
+                ("lpClass", ctypes.c_char_p),
+                ("hkeyClass", ctypes.c_void_p),
+                ("dwHotKey", ctypes.c_ulong),
+                ("hIconOrMonitor", ctypes.c_void_p),
+                ("hProcess", ctypes.c_void_p),
+            ]
+
+        sei = SHELLEXECUTEINFO()
+        sei.cbSize = ctypes.sizeof(sei)
+        sei.fMask = SEE_MASK_NO_CONSOLE
+        sei.lpVerb = b"runas"
+        sei.lpFile = bytes(tempUpdaterPath)
+        sei.lpParameters = f'"{updateTempPath}" "{baseSoftwarePath}"'.encode('utf-8')
+        sei.nShow = 1
+
+        if not ShellExecuteEx(ctypes.byref(sei)):
+            raise ctypes.WinError()
+
+    elif system == 'Linux':
+        updaterPath = Path(baseDir) / "UpdaterUtil"
+
+        tempUpdaterPath = Path(tempDir) / updaterPath.name
+        shutil.copy(updaterPath, tempUpdaterPath)
+
+        subprocess.Popen(
+            f'pkexec "{tempUpdaterPath}" "{updateTempPath}" "{baseSoftwarePath}"',
+            start_new_session=True,
+            close_fds=True,
+            shell=True,
+        )

Installer/Installer.py

@@ -7,6 +7,7 @@ import os
 import platform
 import subprocess
 import tempfile
+import stat
 from pathlib import Path
 
 import requests
@@ -14,19 +15,17 @@ import py7zr
 from PySide6 import QtCore, QtWidgets, QtGui
 
 # Requirements:
-#   requests
-#   PySide6
-#   py7zr
+#   requests PySide6 py7zr wheel pip nuitka ordered-set zstandard
 #
-# Compile with (requires zstandard, benefits from orderedset):
+# Compile with:
 #
 # Windows:
 """
 python -m nuitka --follow-imports --onefile --noinclude-pytest-mode=nofollow --noinclude-setuptools-mode=nofollow ^
 --noinclude-custom-mode=setuptools:error --noinclude-IPython-mode=nofollow --enable-plugin=pyside6 ^
---assume-yes-for-downloads --remove-output --disable-console --warn-unusual-code --show-modules ^
---windows-company-name="AccentuSoft" --windows-product-name="LinkScope Installer" --windows-product-version=2.0.0.0 ^
---include-data-files="Icon.ico=Icon.ico" --linux-icon="Icon.ico" --windows-icon-from-ico=".\Icon.ico" ^
+--assume-yes-for-downloads --remove-output --windows-console-mode=disable --warn-unusual-code --show-modules ^
+--windows-company-name="AccentuSoft" --windows-product-name="LinkScope Installer" --windows-product-version=1.6.5.0 ^
+--include-data-files="Icon.ico=Icon.ico" --windows-icon-from-ico=".\Icon.ico" ^
 --windows-file-description="LinkScope Installer" ^
 Installer.py
 """
@@ -35,7 +34,7 @@ Installer.py
 """
 python -m nuitka --follow-imports --onefile --noinclude-pytest-mode=nofollow --noinclude-setuptools-mode=nofollow \
 --noinclude-custom-mode=setuptools:error --noinclude-IPython-mode=nofollow --enable-plugin=pyside6 \
---assume-yes-for-downloads --remove-output --disable-console --warn-unusual-code --show-modules \
+--assume-yes-for-downloads --remove-output --warn-unusual-code --show-modules \
 --include-data-files="Icon.ico=Icon.ico" --linux-icon="Icon.ico" \
 Installer.py
 """
@@ -717,6 +716,117 @@ For more information on this, and how to apply and follow the GNU AGPL, see
 """
 
 
+def recursiveMakeWriteableHelper(targetPath: Path):
+    for root, dirs, files in os.walk(targetPath):
+        for dir_name in dirs:
+            dir_path = os.path.join(root, dir_name)
+            os.chmod(dir_path, stat.S_IWRITE)
+            if not os.access(dir_path, os.W_OK):
+                os.chmod(dir_path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            os.chmod(file_path, stat.S_IWRITE)
+            if not os.access(file_path, os.W_OK):
+                os.chmod(file_path, stat.S_IWRITE | stat.S_IREAD)
+
+def deleteVenvStuff() -> None:
+    baseAppStoragePath = Path(
+        QtCore.QStandardPaths.standardLocations(
+            QtCore.QStandardPaths.StandardLocation.AppDataLocation)[0])
+    recursiveMakeWriteableHelper(baseAppStoragePath)
+    shutil.rmtree(baseAppStoragePath)
+
+
+def installGraphviz() -> None:
+    if platform.system() == 'Linux':
+        installGraphvizLinuxHelper()
+    else:
+        installGraphvizWindowsHelper()
+
+
+def installGraphvizLinuxHelper():
+    subprocess.run(['apt', 'update'])
+    # https://doc.qt.io/qt-6/linux-requirements.html
+    # https://github.com/Nuitka/Nuitka/issues/2138
+    # No need to check if this succeeds - if there are any issues with installation, we will throw
+    #   an error on the install command.
+    command = subprocess.run(['apt', 'install', 'p7zip-full', 'libopengl0', 'graphviz', 'libmagic1',
+                              'libfontconfig1-dev', 'libfreetype6-dev', 'libatspi2.0-dev',
+                              'libcairo2-dev', 'python3-dev', 'pkg-config', '-y'])
+    if command.returncode != 0:
+        raise ValueError('Installing new packages failed, cannot continue installation.')
+
+
+def installGraphvizWindowsHelper():
+    graphVizPage = requests.get('https://graphviz.org/download/')
+    graphVizParts = graphVizPage.text.split('\n')
+    graphVizDownloadLink = next((chunk.split('"')[1] for chunk in graphVizParts
+                                 if '(64-bit) EXE installer' in chunk), "")
+
+    if not isinstance(graphVizDownloadLink, str) or graphVizDownloadLink == "":
+        raise ValueError('Cannot install GraphViz: Failed to locate the latest version of the GraphViz installer.')
+    graphVizInstallerFileHandler, graphVizInstallerTemp = tempfile.mkstemp()
+    tempPath = Path(graphVizInstallerTemp)
+
+    with os.fdopen(graphVizInstallerFileHandler, 'wb') as tempInstallerFile:
+        with requests.get(graphVizDownloadLink, stream=True) as fileStream:
+            for chunk in fileStream.iter_content(chunk_size=5 * 1024 * 1024):
+                tempInstallerFile.write(chunk)
+    tempPath.chmod(tempPath.stat().st_mode | 0o111)
+    subprocess.run([str(tempPath)])
+    tempPath.unlink(missing_ok=True)
+
+
+def installPythonLinuxHelper():
+    subprocess.run('echo "y" | add-apt-repository ppa:deadsnakes/ppa', shell=True)
+    subprocess.run('apt-get update && apt-get install python3.13 python3.13-venv -y', shell=True)
+    subprocess.run('wget -O - https://bootstrap.pypa.io/get-pip.py | python3.13', shell=True)
+
+
+def installPythonWindowsHelper():
+    win_downloads = requests.get('https://www.python.org/downloads/windows/')
+
+    if win_downloads.status_code != 200:
+        return False
+
+    latest_release_path = win_downloads.text.split(
+        'Python 3.13.', 1)[0].split('href="')[-1].split('"', 1)[0]
+
+    latest_release_full_path = f'https://www.python.org{latest_release_path}'
+
+    latest_release_page = requests.get(latest_release_full_path)
+
+    if latest_release_page.status_code != 200:
+        return False
+
+    latest_release_installer_path = latest_release_page.text.split(
+        'Windows installer (64-bit)', 1)[0].split('href="')[-1].split('"', 1)[0]
+
+    latest_release_binary = requests.get(latest_release_installer_path, allow_redirects=True)
+
+    if latest_release_binary.status_code != 200:
+        return False
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir_path = Path(tmpdir)
+        installer_file_path = tmpdir_path / 'python.exe'
+        installer_file_path.touch(mode=0o777)
+        with open(installer_file_path, 'wb') as file:
+            file.write(latest_release_binary.content)
+        ctypes.windll.shell32.ShellExecuteW(None, "runas", installer_file_path, "/quiet", None, 1)
+
+
+def removeFileHelper(pathToRemove: Path):
+    if not pathToRemove.exists():
+        return
+    if pathToRemove.is_dir():
+        recursiveMakeWriteableHelper(pathToRemove)
+        shutil.rmtree(pathToRemove)
+    else:
+        os.chmod(pathToRemove, stat.S_IWRITE)
+        pathToRemove.unlink(missing_ok=True)
+
+
 class InstallWizard(QtWidgets.QWizard):
 
     # INSTALLER PATHS:
@@ -727,19 +837,10 @@ class InstallWizard(QtWidgets.QWizard):
     #   0 -> 1 -> 4 -> 2 -> 5 -> 6 -> -1
     # Install, Graphviz exists
     #   0 -> 1 -> 4 -> 5 -> 6 -> -1
-    # Update
-    #   0 -> 4 -> 2 -> 5 -> 6 -> -1
-    #          -> 5 -> 6 -> -1
-
     def nextId(self) -> int:
         if self.currentId() == 0:
-            if self.currentPage().uninstallRadio.isChecked():
-                return 3
-            elif self.currentPage().updateRadio.isChecked():
-                return 4
-            else:
-                # Default action is install.
-                return 1
+            # Default action is install.
+            return 3 if self.currentPage().uninstallRadio.isChecked() else 1
         if self.currentId() == 1:
             return 4
         if self.currentId() == 2:
@@ -763,10 +864,7 @@ class InstallWizard(QtWidgets.QWizard):
         self.trayIcon.show()
 
         if len(sys.argv) < 5:
-            latestReleaseDetails = requests.get('https://github.com/AccentuSoft/LinkScope_Client/releases/latest',
-                                                headers={'Accept': 'application/json'})
-            latestVersion = latestReleaseDetails.json()['tag_name']
-            downloadURLBase = f"https://github.com/AccentuSoft/LinkScope_Client/releases/download/{latestVersion}/"
+            downloadURLBase = "https://github.com/AccentuSoft/LinkScope_Client/releases/latest/download/"
 
             if self.currentOS == 'Windows':
                 try:
@@ -785,9 +883,13 @@ class InstallWizard(QtWidgets.QWizard):
                     self.executablePath = self.baseSoftwarePath / 'LinkScope.exe'
                     self.downloadURL = f"{downloadURLBase}LinkScope-Windows-x64.7z"
 
-                    newArgs = ['"' + str(self.desktopShortcutPath) + '"', str(self.graphvizExists),
-                               '"' + str(self.baseSoftwarePath) + '"', '"' + str(self.executablePath) + '"',
-                               '"' + self.downloadURL + '"']
+                    newArgs = [
+                        f'"{str(self.desktopShortcutPath)}"',
+                        str(self.graphvizExists),
+                        f'"{str(self.baseSoftwarePath)}"',
+                        f'"{str(self.executablePath)}"',
+                        f'"{self.downloadURL}"',
+                    ]
                     ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, " ".join(newArgs), None, 1)
                     sys.exit(0)
             elif self.currentOS == 'Linux':
@@ -893,33 +995,18 @@ class InstallWizard(QtWidgets.QWizard):
 
         self.show()
 
-    def removeFileHelper(self, pathToRemove: Path):
-        if not pathToRemove.exists():
-            return
-        pathToRemove.chmod(0o777)
-        if pathToRemove.is_dir():
-            for dirpath, dirnames, filenames in os.walk(pathToRemove):
-                Path(dirpath).chmod(0o777)
-                filenames.extend(dirnames)
-                for filename in filenames:
-                    filePath = Path(dirpath) / filename
-                    filePath.chmod(0o777)
-            shutil.rmtree(pathToRemove)
-        else:
-            pathToRemove.unlink(missing_ok=True)
-
     def createShortcut(self):
-        self.removeFileHelper(self.desktopShortcutPath)
+        removeFileHelper(self.desktopShortcutPath)
         if self.currentOS == 'Linux':
             QtCore.QFile.link(str(self.appPath), str(self.desktopShortcutPath))
         elif self.currentOS == 'Windows':
             QtCore.QFile.link(str(self.executablePath), str(self.desktopShortcutPath))
 
     def uninstall(self):
-        self.removeFileHelper(self.desktopShortcutPath)
+        removeFileHelper(self.desktopShortcutPath)
         if self.currentOS == 'Linux':
-            self.removeFileHelper(self.appPath)
-        self.removeFileHelper(self.baseSoftwarePath)
+            removeFileHelper(self.appPath)
+        removeFileHelper(self.baseSoftwarePath)
 
     def downloadClient(self):
         if not isinstance(self.downloadURL, str) or self.downloadURL == "":
@@ -937,58 +1024,39 @@ class InstallWizard(QtWidgets.QWizard):
 
         tempPath.unlink(missing_ok=True)
 
-    def downloadGraphviz(self):
-        graphVizPage = requests.get('https://graphviz.org/download/')
-        graphVizParts = graphVizPage.text.split('\n')
-        graphVizDownloadLink = next((chunk.split('"')[1] for chunk in graphVizParts
-                                     if '(64-bit) EXE installer' in chunk), "")
-
-        if not isinstance(graphVizDownloadLink, str) or graphVizDownloadLink == "":
-            raise ValueError('Cannot install GraphViz: Failed to locate the latest version of the GraphViz installer.')
-        graphVizInstallerTemp = tempfile.mkstemp()
-        tempPath = Path(graphVizInstallerTemp[1])
-
-        with os.fdopen(graphVizInstallerTemp[0], 'wb') as tempInstallerFile:
-            with requests.get(graphVizDownloadLink, stream=True) as fileStream:
-                for chunk in fileStream.iter_content(chunk_size=5 * 1024 * 1024):
-                    tempInstallerFile.write(chunk)
-        tempPath.chmod(tempPath.stat().st_mode | 0o111)
-        subprocess.run([str(tempPath)])
-        tempPath.unlink(missing_ok=True)
-
     def install(self):
         # Assumes we have superuser privileges.
-        if self.currentOS != 'Linux':
-            # Assume the user has installed / will install Graphviz.
-            # We don't actually need to do anything here. Maybe in the future, register application in registry?
-            return
-        # Redundant since we always try to update and install, but it's good coding practice.
-        if not self.graphvizExists:
-            # No need to check if this succeeds - if there are any issues with installation, we will throw
-            #   an error on the install command.
-            subprocess.run(['apt', 'update'])
-            command = subprocess.run(['apt', 'install', 'p7zip-full', 'libopengl0', 'graphviz', 'libmagic1', '-y'])
-            if command.returncode != 0:
-                raise ValueError('Installing new packages failed, cannot continue installation.')
+        # We do not install/configure anything on the virtual env, the first time boot will take longer, but
+        #   it's safer to do it like this since we don't know how the user's system is configured.
 
-        self.removeFileHelper(self.appPath)
-        with open(self.appPath, 'w') as desktopApplicationFile:
-            desktopApplicationFile.write(LINUX_DESKTOP_FILE_ENTRY)
-        # Mark desktop file as executable
-        self.appPath.chmod(self.appPath.stat().st_mode | 0o111)
+        if self.currentOS == 'Linux':
+            try:
+                if subprocess.run(["python3.13", "--version"],
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE,
+                                  text=True).returncode != 0:
+                    raise ValueError()
+            except Exception:
+                installPythonLinuxHelper()
+        elif not subprocess.check_output(['where', 'python']):
+            installPythonWindowsHelper()
+
+        if not self.graphvizExists or self.currentOS == 'Linux':
+            installGraphviz()
+
+        if self.currentOS == 'Linux':
+            removeFileHelper(self.appPath)
+            with open(self.appPath, 'w') as desktopApplicationFile:
+                desktopApplicationFile.write(LINUX_DESKTOP_FILE_ENTRY)
+            # Mark desktop file as executable
+            self.appPath.chmod(self.appPath.stat().st_mode | 0o111)
 
 
 class IntroInstallUninstallPage(QtWidgets.QWizardPage):
 
     def validatePage(self) -> bool:
-        if self.updateRadio.isChecked():
-            self.wizard().page(5).setTitle('Updating LinkScope Installation')
-            self.wizard().page(5).installProgressLabel.setText('Update Process: ')
-            self.wizard().page(5).updateSelected = True
-        else:
-            self.wizard().page(5).setTitle('Install LinkScope')
-            self.wizard().page(5).installProgressLabel.setText('Installation Process: ')
-            self.wizard().page(5).updateSelected = False
+        self.wizard().page(5).setTitle('Install LinkScope')
+        self.wizard().page(5).installProgressLabel.setText('Installation Process: ')
         return True
 
     def __init__(self):
@@ -1007,10 +1075,6 @@ class IntroInstallUninstallPage(QtWidgets.QWizardPage):
         self.installRadio.setToolTip('Install the LinkScope Client software.')
         self.installRadio.setChecked(True)
 
-        self.updateRadio = QtWidgets.QRadioButton('Update / Repair LinkScope')
-        self.updateRadio.setToolTip('Update the existing installation of LinkScope Client, or repair any issues with '
-                                    'the existing installation.')
-
         self.uninstallRadio = QtWidgets.QRadioButton('Uninstall LinkScope')
         self.uninstallRadio.setToolTip('Uninstall the LinkScope Client software.\nNote that on Linux, any packages '
                                        'that were installed during the installation of LinkScope will not be removed.\n'
@@ -1018,7 +1082,6 @@ class IntroInstallUninstallPage(QtWidgets.QWizardPage):
                                        'on them.')
 
         installUninstallLayout.addWidget(self.installRadio)
-        installUninstallLayout.addWidget(self.updateRadio)
         installUninstallLayout.addWidget(self.uninstallRadio)
 
 
@@ -1077,7 +1140,6 @@ class LinkScopeInstallLatestPage(QtWidgets.QWizardPage):
                                              'LinkScope. Click "Commit" to start the installation.')
 
         self.createShortcut = False
-        self.updateSelected = False
         self.processStarted = False
         self.installThread = None
 
@@ -1109,8 +1171,9 @@ class LinkScopeInstallLatestPage(QtWidgets.QWizardPage):
 class LinkScopeUninstallPage(QtWidgets.QWizardPage):
 
     def doStuff(self):
-        self.progressBar.setValue(3)
         try:
+            deleteVenvStuff()
+            self.progressBar.setValue(3)
             self.wizard().uninstall()
         except Exception as e:
             self.wizard().page(6).doneLabel.setText('Error occurred during uninstallation: ' +
@@ -1215,8 +1278,6 @@ class LicensePage(QtWidgets.QWizardPage):
         rejectLicense.setChecked(True)
 
         self.registerField('Accept Terms*', self.acceptLicense)
-        self.setMinimumWidth(550)
-        self.setMinimumHeight(600)
 
         licenseLayout.addWidget(licenseLabel)
         licenseLayout.addWidget(licenseText)
@@ -1250,22 +1311,15 @@ class InstallThread(QtCore.QThread):
 
     def run(self) -> None:
         try:
-            # We have already installed Graphviz on Linux at this point.
-            if not self.wizard.graphvizExists and self.wizard.currentOS == 'Windows':
-                self.progressSignal.emit(1)
-                self.wizard.downloadGraphviz()
             self.progressSignal.emit(2)
-            if self.installPage.updateSelected:
-                self.wizard.uninstall()
             self.wizard.install()
             self.installPage.downloadingLabel.setVisible(True)
             self.installPage.downloadingLabel.setHidden(False)
             self.progressSignal.emit(4)
             self.wizard.downloadClient()
             self.progressSignal.emit(8)
-            shortcutExists = self.wizard.desktopShortcutPath.exists()
             self.progressSignal.emit(9)
-            if self.installPage.createShortcut or (self.installPage.updateSelected and shortcutExists):
+            if self.installPage.createShortcut:
                 self.wizard.createShortcut()
             self.progressSignal.emit(10)
             self.doneSignal.emit(True)
@@ -1278,5 +1332,7 @@ class InstallThread(QtCore.QThread):
 
 if __name__ == '__main__':
     application = QtWidgets.QApplication(sys.argv)
+    application.setOrganizationName("AccentuSoft")
+    application.setApplicationName("LinkScope Client")
     installWizard = InstallWizard()
     sys.exit(application.exec())

Modules/Aircraft_Inquiry/AircraftInquiryByDealer.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByDealer:
-    name = "Aircraft Inquiry By Dealer"
-    category = "Aircraft"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Company"}
-    resultTypes = {'Phrase', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import pandas as pd
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}DealerResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Dealertxt": entity['Company Name']}))
-            for future in as_completed(futures):
-                uid = uidList[futures.index(future)]
-                try:
-                    df_list = pd.read_html(future.result().text)
-                except requests.exceptions.ConnectionError:
-                    return "Please check your internet connection"
-                except ValueError:
-                    return_result = []
-                    return return_result
-                df = df_list[0]
-                for certificate_index in range(len(df["Certificate Number"])):
-                    index_of_child = len(return_result)
-                    return_result.append([{'Company Name': df["Name"][certificate_index],
-                                           'Entity Type': 'Company'},
-                                          {uid: {'Resolution': 'Aircraft Dealer', 'Notes': ''}}])
-                    return_result.append([{'Phrase': df["Certificate Number"][certificate_index],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Aircraft Certificate Number', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByEngine.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByEngine:
-    name = "Aircraft Inquiry By Engine"
-    category = "Aircraft"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Phrase"}
-    resultTypes = {"Phrase"}
-    parameters = {'Manufacturer': {'description': "Enter the Manufacturer of the Engine Model",
-                                   'type': 'String',
-                                   'value': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        Manufacturer = parameters['Manufacturer']
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}EngineReferenceResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Modeltxt": entity['Phrase'],
-                                                               "MfrNametxt": Manufacturer}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            return_result.append([{'Phrase': f"Model Code:str({df['Mfr/Mdl Code']})",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Model Code', 'Notes': ''}}])
-            return_result.append([{'Phrase': f"Engine Type:{df['Type Engine']}",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Engine Type', 'Notes': ''}}])
-            return_result.append([{'Phrase': f"Horse Power:str({df['Horsepower']})",
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'Aircraft Engine Horsepower', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByNNumber.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByNNumber:
-    name = "Aircraft Inquiry By N-Number"
-    category = "Aircraft"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Phrase"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company', 'Country', 'City'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}NNumberResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"NNumbertxt": entity['Phrase']}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df1 = df_list[0]
-            df2 = df_list[1]
-            df3 = df_list[2]
-            return_result.append([{'ID Number': df1[1][0],
-                                   'Entity Type': 'Identification Number'},
-                                  {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
-            return_result.append([{'Company Name': df1[1][1],
-                                   'Entity Type': 'Company'},
-                                  {uid: {'Resolution': 'Aircraft Company', 'Notes': ''}}])
-            return_result.append([{'Full Name': df2[1][0],
-                                   'Entity Type': 'Person'},
-                                  {uid: {'Resolution': 'Aircraft Owner', 'Notes': ''}}])
-            return_result.append([{'City Name': df2[1][2],
-                                   'Entity Type': 'City'},
-                                  {uid: {'Resolution': "Aircraft Owner's City", 'Notes': ''}}])
-            return_result.append([{'Country Name': df2[1][4],
-                                   'Entity Type': 'Country'},
-                                  {uid: {'Resolution': "Aircraft Owner's Country", 'Notes': ''}}])
-            return_result.append([{'Phrase': df3[1][1],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': "Aircraft Engine Series", 'Notes': ''}}])
-            return_result.append([{'Phrase': df3[1][2],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': "Aircraft Engine Motor", 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryByPersonName.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryByPersonName:
-    name = "Aircraft Inquiry By Person Name"
-    category = "Aircraft"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Person"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}NameResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"nametxt": entity['Full Name'], "sort_option": "1"}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            for i in range(len(df["N-Number"])):
-                return_result.append([{'Phrase': df["N-Number"][0],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
-                return_result.append([{'ID Number': str(df['Serial Number'][0]),
-                                       'Entity Type': 'Identification Number'},
-                                      {uid: {'Resolution': 'Aircraft Identification Number', 'Notes': ''}}])
-                return_result.append([{'Company Name': df['Manufacturer Name Model'][0],
-                                       'Entity Type': 'Company'},
-                                      {uid: {'Resolution': 'Aircraft Manufacturer Name', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/AircraftInquiryBySerialNumber.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-
-class AircraftInquiryBySerialNumber:
-    name = "Aircraft Inquiry By Serial Number"
-    category = "Aircraft"
-    description = "Find information about aircraft identifications from https://registry.faa.gov/aircraftinquiry/"
-    originTypes = {"Identification Number"}
-    resultTypes = {'Phrase', 'Person', 'Identification Number', 'Company'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-        import pandas as pd
-
-        futures = []
-        uidList = []
-        return_result = []
-
-        submit_url = "https://registry.faa.gov/aircraftinquiry/Search/"
-        crafted_url = f"{submit_url}SerialResult"
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                futures.append(session.post(crafted_url, data={"Serialtxt": entity['ID Number'], "sort_option": "1"}))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                df_list = pd.read_html(future.result().text)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            except ValueError:
-                return "No results retrieved"
-            df = df_list[0]
-            for i in range(len(df["N-Number"])):
-                return_result.append([{'Company Name': df["Manufacturer Name"][i],
-                                       'Entity Type': 'Company'},
-                                      {uid: {'Resolution': 'Aircraft Manufacturer', 'Notes': ''}}])
-                return_result.append([{'Phrase': df["N-Number"][i],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft N-Number', 'Notes': ''}}])
-                return_result.append([{'Phrase': df["Model"][i],
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aircraft Model', 'Notes': ''}}])
-        return return_result

Modules/Aircraft_Inquiry/requirements.txt

@@ -1,3 +0,0 @@
-requests
-pandas
-requests-futures

Modules/Aleph/Aleph.xml

@@ -1,18 +0,0 @@
-<Aleph>
-  <Aleph_ID>
-    <Attributes>
-      <Attribute default="Aleph Default ID" check="String" primary="True">ID</Attribute>
-    </Attributes>
-    <Icon>
-      Default.svg
-    </Icon>
-  </Aleph_ID>
-  <Aleph_Collection_ID>
-    <Attributes>
-      <Attribute default="Aleph Default Collection ID" check="String" primary="True">ID</Attribute>
-    </Attributes>
-    <Icon>
-      Default.svg
-    </Icon>
-  </Aleph_Collection_ID>
-</Aleph>

Modules/Aleph/Aleph_Entity_Search.py

@@ -1,273 +0,0 @@
-#!/usr/bin/env python3
-
-
-class Aleph_Entity_Search:
-    name = "Aleph Entity Search"
-    category = "Aleph OCCRP"
-    description = "Find information about a given search parameter"
-    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
-    resultTypes = {'Phrase'}
-    parameters = {'Max Results': {'description': 'The maximum number of results to return.',
-                                  'type': 'String',
-                                  'value': 'Enter the number of results you want returned',
-                                  'default': '1'},
-                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}
-                  }
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import pycountry
-        import time
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        return_result = []
-        uidList = []
-        futures = []
-
-        url = "https://aleph.occrp.org/api/2/entities"
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        gender = "None"
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        try:
-            max_results = int(parameters['Max Results'])
-        except ValueError:
-            return "The value for parameter 'Max Results' is not a valid integer."
-
-        if max_results <= 0:
-            return []
-
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                crafted_url = url + f"?q={primary_field}&filter:schemata=Thing&limit={max_results}"
-                time.sleep(1)
-                futures.append(session.get(crafted_url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            # print(response)
-            for schema in response['results']:
-                index_of_child = len(return_result)
-                try:
-                    if schema['schema'] == "Person":
-                        if schema['properties'].get('gender') is not None \
-                                and schema['properties'].get('gender')[0] == "F":
-                            gender = "Female"
-                        elif schema['properties'].get('gender') is not None \
-                                and schema['properties'].get('gender')[0] == "M":
-                            gender = "Male"
-                        if schema['properties'].get('legalForm') is not None:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties'].get('birthDate')),
-                                  'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}",
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Full Name': str(schema['properties']['name'][0]),
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['birthDate'][0]),
-                                  'Notes': schema['links']['self'],
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Person Entity', 'Notes': ''}}])
-                        if schema['properties'].get('registrationNumber') is not None:
-                            return_result.append(
-                                [{'Registration Number': str(schema['properties']['registrationNumber'][0]),
-                                  'Notes': '',
-                                  'Entity Type': 'Company'},
-                                 {index_of_child: {'Resolution': 'Aleph Registration Number', 'Notes': ''}}])
-                        if schema['properties'].get('country') is not None:
-                            return_result.append(
-                                [{'Country Name': str(
-                                    pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
-                                    'Notes': '',
-                                    'Entity Type': 'Country'},
-                                    {index_of_child: {'Resolution': 'Country of Origin', 'Notes': ''}}])
-                        if schema['properties'].get('addressEntity'):
-                            return_result.append(
-                                [{'Street Address': str(
-                                    schema['properties']['addressEntity'][0]['properties']['full'][0]),
-                                    'Notes': '',
-                                    'Entity Type': 'Address'},
-                                    {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Street Address': str(schema['properties']['address'][0]),
-                                  'Notes': '',
-                                  'Entity Type': 'Address'},
-                                 {index_of_child: {'Resolution': 'Address Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph Collection ID'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                    elif schema['schema'] == "Organization":
-                        return_result.append(
-                            [{'Organization Name': str(schema['properties']['name'][0]),
-                              'Registration Number': str(schema['properties']['registrationNumber'][0]),
-                              'Notes': f"{schema['links']['self']}\nLegal Form: {schema['properties']['legalForm'][0]}\n"
-                                       f"Source URL: {schema['properties']['sourceUrl'][0]}",
-                              'Entity Type': 'Organization'},
-                             {uid: {'Resolution': 'Aleph Organisation Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Country Name': str(
-                                pycountry.countries.get(alpha_2=schema['properties']['country'][0]).name),
-                                'Notes': '',
-                                'Entity Type': 'Country'},
-                                {index_of_child: {'Resolution': "Aleph Organisation Country", 'Notes': ''}}])
-                        return_result.append(
-                            [{'Street Address': str(schema['properties']['address'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Address'},
-                             {index_of_child: {'Resolution': "Aleph Organisation Address", 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': "Aleph Organisation ID", 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phone Number': str(schema['properties']['phone'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Phone Number'},
-                             {index_of_child: {'Resolution': 'Phone Number', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['classification'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Organisation Classification', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                    elif schema['schema'] == "Pages":
-                        if 'updated_at' in schema:
-                            date_created = schema['updated_at']
-                        else:
-                            date_created = schema['created_at']
-                        doc_name = 'Document: ' + schema['properties']['title'][0]
-                        entity_link = schema['links']['self']
-                        file_link = schema['links']['file']
-                        source_url = schema['properties']['sourceUrl'][0]
-                        return_result.append(
-                            [{'Phrase': doc_name,
-                              'Source': source_url,
-                              'Notes': 'Link to Aleph Entity: ' + entity_link + '\n\n' +
-                                       'Link to document: ' + file_link,
-                              'Entity Type': 'Phrase',
-                              'Date Created': date_created},
-                             {uid: {'Resolution': 'Aleph Document', 'Notes': ''}}])
-                    elif schema['properties']['parent'][0]['schema'] == "Person":
-                        gender = str(schema['properties']['parent'][0]['properties'].get('gender')[0])
-                        if schema['properties']['parent'][0]['properties'].get('legalForm') is not None:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
-                                  'Notes': f"{schema['properties']['parent'][0]['links']['self']}\nLegal Form: "
-                                           f"{schema['properties']['parent'][0]['properties']['legalForm'][0]}",
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
-                        else:
-                            return_result.append(
-                                [{'Full Name': schema['properties']['parent'][0]['properties']['name'][0],
-                                  'Gender': gender,
-                                  'Date Of Birth': str(schema['properties']['parent'][0]['properties']['birthDate'][0]),
-                                  'Notes': schema['properties']['parent'][0]['links']['self'],
-                                  'Entity Type': 'Person'},
-                                 {uid: {'Resolution': 'Aleph Person Entity', 'Notes': ''}}])
-                        if schema['properties']['parent'][0]['properties'].get('registrationNumber') is not None:
-                            return_result.append(
-                                [{'Registration Number': str(
-                                    schema['properties']['parent'][0]['properties']['registrationNumber'][0]),
-                                    'Notes': '',
-                                    'Entity Type': 'Company'},
-                                    {index_of_child: {'Resolution': 'Company Registration Number', 'Notes': ''}}])
-                        if schema['properties']['parent'][0]['properties'].get('country') is not None:
-                            return_result.append(
-                                [{'Country Name': str(
-                                    pycountry.countries.get(
-                                        alpha_2=schema['properties']['parent'][0]['properties']['country'][0]).name),
-                                    'Notes': '',
-                                    'Entity Type': 'Country'},
-                                    {index_of_child: {'Resolution': 'Country', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['properties']['parent'][0]['id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph ID'},
-                             {index_of_child: {'Resolution': 'Aleph ID', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['label']),
-                              'Notes': str(schema['properties']['parent'][0]['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['properties']['parent'][0]['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Phrase'},
-                             {index_of_child: {'Resolution': 'Aleph Collection ID', 'Notes': ''}}])
-                        index_of_child_of_child = len(return_result)
-                        return_result.append(
-                            [{'Company Name': str(schema['properties']['name'][0]),
-                              'Notes': schema['links']['self'],
-                              'Entity Type': 'Company'},
-                             {index_of_child: {'Resolution': 'Aleph Company Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Street Address': str(schema['properties']['addressEntity'][0]['properties']['full'][0]),
-                              'Notes': '',
-                              'Entity Type': 'Address'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Person Address', 'Notes': ''}}])
-                        for country_code in schema['collection']['countries']:
-                            return_result.append(
-                                [{'Country Name': str(pycountry.countries.get(alpha_2=country_code).name),
-                                  'Notes': '',
-                                  'Entity Type': 'Country'},
-                                 {index_of_child_of_child: {'Resolution': 'Country', 'Notes': ''}}])
-                        return_result.append(
-                            [{'Phrase': str(schema['collection']['label']),
-                              'Notes': str(schema['collection']['summary']),
-                              'Entity Type': 'Phrase'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Collection Entity', 'Notes': ''}}])
-                        return_result.append(
-                            [{'ID': str(schema['collection']['collection_id']),
-                              'Notes': '',
-                              'Entity Type': 'Aleph Collection ID'},
-                             {index_of_child_of_child: {'Resolution': 'Aleph Entity Search', 'Notes': ''}}])
-                except (TypeError, KeyError):
-                    # print(repr(e))
-                    continue
-        return return_result

Modules/Aleph/GetCollectionByID.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetCollectionByID:
-    name = "Get Collections By ID"
-    category = "Aleph OCCRP"
-    description = "Find information about Collections and their IDs"
-    originTypes = {'Phrase'}
-    resultTypes = {'Phrase. Person, Address, Phone Number, Email Address, Country, Bank Account'}
-    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                url = f"https://aleph.occrp.org/api/2/collections/{primary_field}"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            if response['statistics']['names'].get('values') is not None:
-                nameKeys = list(response['statistics']['names'].get('values').keys())
-                for nameKey in nameKeys:
-                    returnResults.append([{'Full Name': str(nameKey),
-                                           'Entity Type': 'Person'},
-                                          {uid: {'Resolution': 'Person Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['addresses'].get('values') is not None:
-                addressKeys = list(response['statistics']['addresses'].get('values').keys())
-                for addressKey in addressKeys:
-                    returnResults.append([{'Street Address': str(addressKey),
-                                           'Entity Type': 'Address'},
-                                          {uid: {'Resolution': 'Address Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['phones'].get('values') is not None:
-                phoneKeys = list(response['statistics']['phones'].get('values').keys())
-                for phoneKey in phoneKeys:
-                    returnResults.append([{'Phone Number': str(phoneKey),
-                                           'Entity Type': 'Phone Number'},
-                                          {uid: {'Resolution': 'Phone Number Entity',
-                                                 'Notes': ''}}])
-
-            if response['statistics']['emails'].get('values') is not None:
-                emailKeys = list(response['statistics']['emails'].get('values').keys())
-                for emailKey in emailKeys:
-                    returnResults.append([{'Email Address': str(emailKey),
-                                           'Entity Type': 'Email Address'},
-                                          {uid: {'Resolution': 'Email Address Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['countries'].get('values') is not None:
-                countriesKeys = list(response['statistics']['countries'].get('values').keys())
-                for countriesKey in countriesKeys:
-                    returnResults.append([{'Country Name': str(countriesKey),
-                                           'Entity Type': 'Country'},
-                                          {uid: {'Resolution': 'Country Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['languages'].get('values') is not None:
-                languagesKeys = list(response['statistics']['languages'].get('values').keys())
-                for languagesKey in languagesKeys:
-                    returnResults.append([{'Phrase': str(languagesKey),
-                                           'Entity Type': 'Phrase'},
-                                          {uid: {'Resolution': 'Language Entity',
-                                                 'Notes': ''}}])
-            if response['statistics']['ibans'].get('values') is not None:
-                ibansKeys = list(response['statistics']['ibans'].get('values').keys())
-                for ibansKey in ibansKeys:
-                    returnResults.append([{'Account Number': str(ibansKey),
-                                           'Entity Type': 'Bank Account'},
-                                          {uid: {'Resolution': 'IBAN Entity',
-                                                 'Notes': ''}}])
-        return returnResults

Modules/Aleph/GetCollectionsInfo.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetCollectionsInfo:
-    name = "Get Collections Info"
-    category = "Aleph OCCRP"
-    description = "Find information about Collections and their IDs"
-    originTypes = {'Phrase'}
-    resultTypes = {'Phrase, Aleph ID'}
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.',
-                                  'type': 'String',
-                                  'default': '1'},
-                  'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}
-                  }
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-
-        try:
-            maxResults = int(parameters['Max Results'])
-        except ValueError:
-            return "The value for parameter 'Max Results' is not a valid integer."
-        if maxResults <= 0:
-            return []
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                url = f"https://aleph.occrp.org/api/2/collections?offset=0&limit=300&page"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            collections = response['results'][:maxResults]
-
-            for collection in collections:
-                index_of_child = len(returnResults)
-                returnResults.append([{'Phrase': collection['label'],
-                                       'Notes': str(collection.get('summary')),
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': 'Aleph Collection Name',
-                                             'Notes': ''}}])
-
-                returnResults.append([{'ID': collection['id'],
-                                       'Entity Type': 'Aleph ID'},
-                                      {index_of_child: {'Resolution': 'Aleph Collection ID',
-                                                        'Notes': ''}}])
-        return returnResults

Modules/Aleph/GetSimilarEntities.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-
-class GetSimilarEntities:
-    name = "Get Similar Entities"
-    category = "Aleph OCCRP"
-    description = "Find information about similar entities"
-    originTypes = {'Phrase', 'Person', 'Politically Exposed Person'}
-    resultTypes = {'Phrase', 'Person', 'Address', 'Aleph ID'}
-    parameters = {'Aleph Disclaimer': {'description': 'The content on Aleph is provided for general information only.\n'
-                                                      'It is not intended to amount to advice on which you should place'
-                                                      'sole and entire reliance.\n'
-                                                      'We recommend that you conduct your own independent fact checking'
-                                                      'against the data and materials that you access on Aleph.\n'
-                                                      'Aleph API is not a replacement for traditional due diligence '
-                                                      'checks and know-your-customer background checks.',
-                                       'type': 'String',
-                                       'value': 'Type "Accept" (without quotes) to confirm your understanding.',
-                                       'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import time
-        import requests
-        import pycountry
-        from requests_futures.sessions import FuturesSession
-        from concurrent.futures import as_completed
-
-        returnResults = []
-        futures = []
-        uidList = []
-
-        if parameters['Aleph Disclaimer'] != 'Accept':
-            return "Please Accept the Terms for Aleph."
-
-        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
-        with FuturesSession(max_workers=15) as session:
-            for entity in entityJsonList:
-                uidList.append(entity['uid'])
-                primary_field = entity[list(entity)[1]].strip()
-                url = f"https://aleph.occrp.org/api/2/entities/{primary_field}/similar"
-                time.sleep(1)
-                futures.append(session.get(url, headers=headers))
-        for future in as_completed(futures):
-            uid = uidList[futures.index(future)]
-            try:
-                response = future.result().json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            entities = response['results']
-            for schema in entities:
-                if schema['entity']['schema'] == 'Person':
-                    index_of_child = len(returnResults)
-                    returnResults.append([{'Full Name': ' '.join(map(str, schema['entity']['properties']['name'])),
-                                           'Gender': ' '.join(map(str, schema['entity']['properties']['gender'])),
-                                           'Notes': ' '.join(map(str, schema['entity']['properties']['legalForm'])),
-                                           'Entity Type': 'Person'},
-                                          {uid: {'Resolution': 'Person Entity',
-                                                 'Notes': ''}}])
-                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
-                    returnResults.append([{'Street Address': schema['entity']['properties']['addressEntity'][0]
-                                           ['properties']['full'][0],
-                                           'Postal Code': schema['entity']['properties']['addressEntity'][0]
-                                           ['properties']['postalCode'][0],
-                                           'Country': country,
-                                           'Entity Type': 'Address'},
-                                          {index_of_child: {'Resolution': 'Address',
-                                                            'Notes': ''}}])
-                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
-                                           'Notes': schema['entity']['collection']['summary'],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Location in Database',
-                                                            'Notes': ''}}])
-
-                    returnResults.append([{'ID': schema['entity']['id'],
-                                           'Entity Type': 'Aleph ID'},
-                                          {index_of_child: {'Resolution': 'ID in Database',
-                                                            'Notes': ''}}])
-
-                elif schema['entity']['schema'] == 'Company':
-                    index_of_child = len(returnResults)
-                    returnResults.append([{'Company Name': ' '.join(map(str, schema['entity']['properties']['name'])),
-                                           'Notes': str(schema['entity']['properties']['status']),
-                                           'Entity Type': 'Company'},
-                                          {uid: {'Resolution': 'Company Entity',
-                                                 'Notes': ''}}])
-                    country = pycountry.countries.get(alpha_2=schema['entity']['properties']['country'][0]).name
-                    returnResults.append([{'Street Address': str(schema['entity']['properties'].get('address')),
-                                           'Country': country,
-                                           'Entity Type': 'Address'},
-                                          {index_of_child: {'Resolution': 'Address',
-                                                            'Notes': ''}}])
-                    returnResults.append([{'Phrase': schema['entity']['collection']['label'],
-                                           'Notes': schema['entity']['collection']['summary'],
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Location in Database',
-                                                            'Notes': ''}}])
-
-                    returnResults.append([{'ID': schema['entity']['id'],
-                                           'Entity Type': 'Aleph ID'},
-                                          {index_of_child: {'Resolution': 'ID in Database',
-                                                            'Notes': ''}}])
-        return returnResults

Modules/Aleph/requirements.txt

@@ -1,3 +0,0 @@
-requests
-pycountry
-requests-futures

Modules/Amass/Amass_Domain.py

@@ -1,280 +0,0 @@
-#!/usr/bin/env python3
-# User needs to be in docker group or to have root privileges
-
-class Amass_Domain:
-    name = "Amass Domain Scan"
-    category = "Network Infrastructure"
-    description = "Find information about a particular domain. Requires Docker to be installed."
-    originTypes = {'Domain'}
-    resultTypes = {'IP Address', 'Phrase', 'Autonomous System', 'Domain', 'IPv6 Address'}
-    parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://virustotal.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'AlienVault API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://otx.alienvault.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://app.binaryedge.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'C99 API Key': {'description': 'Enter your api key under your profile after'
-                                                 ' signing up on https://c99.nl.',
-                                  'type': 'String',
-                                  'value': 'None',
-                                  'global': True,
-                                  'default': 'None'},
-                  'Censys API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://censys.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Chaos API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://chaos.projectdiscovery.io.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://cloudflare.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'DNSDB API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://dnsdb.info.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'GitHub API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://github.com.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Hunter API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://hunter.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
-                                                         ' signing up on https://ipinfo.io.',
-                                          'type': 'String',
-                                          'value': 'None',
-                                          'global': True,
-                                          'default': 'None'},
-                  'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://networksdb.io.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
-                                                          ' signing up on https://passivetotal.com .',
-                                           'type': 'String',
-                                           'value': 'None',
-                                           'global': True,
-                                           'default': 'None'},
-                  'ReconDev API Key': {'description': 'Enter your api key under your profile after'
-                                                      ' signing up on https://recon.dev.',
-                                       'type': 'String',
-                                       'value': 'None',
-                                       'global': True,
-                                       'default': 'None'},
-                  'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
-                                                            ' signing up on https://securitytrails.com.',
-                                             'type': 'String',
-                                             'value': 'None',
-                                             'global': True,
-                                             'default': 'None'},
-                  'Shodan API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://shodan.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Spyse API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://spyse.com.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://threatbook.cn.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'Umbrella API Key': {'description': 'Enter your api key under your profile after'
-                                                      ' signing up on https://umbrella.cisco.com.',
-                                       'type': 'String',
-                                       'value': 'None',
-                                       'global': True,
-                                       'default': 'None'},
-                  'URLScan API Key': {'description': 'Enter your api key under your profile after'
-                                                     ' signing up on https://urlscan.io.',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
-                                                         ' signing up on https://whoisxmlapi.com.',
-                                          'type': 'String',
-                                          'value': 'None',
-                                          'global': True,
-                                          'default': 'None'},
-                  'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://zetalytics.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
-                                                     'in between',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
-                                                        'in between. Obtain them at https://developer.facebook.com',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
-                                                     'in between. Obtain them at https://developer.twitter.com',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'ReconDev.free API Key': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None',
-                      'global': True,
-                      'default': 'None'},
-                  'ReconDev.paid API Key': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None',
-                      'global': True,
-                      'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        import json
-        from ipaddress import ip_address, IPv4Address, IPv6Address
-        import docker
-        import tempfile
-        from docker.errors import APIError
-
-        return_result = []
-        # Generate Config as a temporary file:
-        with tempfile.TemporaryDirectory() as tempDir:
-            tempPath = Path(tempDir).absolute()
-            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                 suffix='Config',
-                                                 dir=tempPath)
-            config.write("share = true\n")
-            config.write("[scope]\n")
-            config.write("port = 80\n")
-            config.write("port = 443\n")
-            config.write("[data_sources]\n")
-            config.write("minimum_ttl = 1440\n")
-            for parameter in self.parameters:
-                if parameters[f'{parameter}'] != 'None':
-                    field1 = f"[data_sources.{parameter}]"
-                    field2 = f"[data_sources.{parameter}.Credentials]"
-                    if parameter == "ZoomEye":
-                        username, password = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"{field2}\n")
-                        config.write(f"username = {username}\n")
-                        config.write(f"password = {password}\n")
-                    elif parameter == "FacebookCT":
-                        field3, secret = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.app1\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                        config.write(f"secret = {secret}\n")
-                    elif parameter == "Twitter":
-                        field3, secret = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.account1\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                        config.write(f"secret = {secret}\n")
-                    elif parameter == "ReconDev.paid":
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.paid\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                    elif parameter == "ReconDev.free":
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.free\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                    else:
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"{field2}\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-            path_to_config = Path(config.name).name
-            config.seek(0)
-            for entity in entityJsonList:
-                primary_field = entity["Domain Name"].strip()
-                try:
-                    client = docker.from_env()
-                    container = client.containers.run("caffix/amass:latest",
-                                                      f"enum -src -d {primary_field} "
-                                                      f"-config /.config/amass/{path_to_config}",
-                                                      volumes={
-                                                          str(tempPath): {'bind': '/.config/amass',
-                                                                          'mode': 'rw'}},
-                                                      remove=True)
-                    jsonFile = tempPath / 'amass.json'
-                    jsonContents = ""
-                    if jsonFile.exists():
-                        with open(jsonFile, 'r') as jsonFileHandler:
-                            jsonContents = jsonFileHandler.read()
-                    client.close()
-                except (APIError, docker.errors.ContainerError) as error:
-                    return "Something happened to the docker container - Cannot continue: " + str(error)
-                uid = entity['uid']
-                for dictionary in jsonContents.splitlines():
-                    index_of_child = len(return_result)
-                    line_dictionary = json.loads(dictionary)
-                    size = len(line_dictionary['addresses'])
-                    return_result.append([{'Domain Name': str(line_dictionary['name']),
-                                           'Entity Type': 'Domain'},
-                                          {uid: {'Resolution': 'Amass Domain Scan', 'Notes': ''}}])
-                    for ip in range(size):
-                        if type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv4Address:
-                            return_result.append([{'IP Address': str(line_dictionary['addresses'][ip]['ip']),
-                                                   'Entity Type': 'IP Address'},
-                                                  {index_of_child: {'Resolution': 'Amass IP Address', 'Notes': ''}}])
-                        elif type(ip_address(line_dictionary['addresses'][ip]['ip'])) is IPv6Address:
-                            return_result.append([{'IPv6 Address': str(line_dictionary['addresses'][ip]['ip']),
-                                                   'Entity Type': 'IPv6 Address'},
-                                                  {index_of_child: {'Resolution': 'Amass IPv6 Address', 'Notes': ''}}])
-                        return_result.append([{'AS Number': "AS" + str(line_dictionary['addresses'][ip]['asn']),
-                                               'ASN Cidr': str(line_dictionary['addresses'][ip]['cidr']),
-                                               'Entity Type': 'Autonomous System'},
-                                              {index_of_child: {'Resolution': 'Amass Autonomous System', 'Notes': ''}}])
-                        return_result.append([{'Phrase': str(line_dictionary['addresses'][ip]['desc']),
-                                               'Entity Type': 'Phrase'},
-                                              {index_of_child: {'Resolution': 'Amass Domain Scan Description',
-                                                                'Notes': ''}}])
-            config.close()
-        return return_result

Modules/Amass/Amass_Intel.py

@@ -1,285 +0,0 @@
-#!/usr/bin/env python3
-# User needs to be in docker group or have root privileges
-
-class Amass_Intel:
-    name = "Amass Intel Scan"
-    category = "Network Infrastructure"
-    description = "Find information about a particular domain. Requires Docker to be installed."
-    originTypes = {'Domain', 'IP Address', 'Autonomous System'}
-    resultTypes = {'Domain'}
-    parameters = {'VirusTotal API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://virustotal.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'AlienVault API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://otx.alienvault.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'BinaryEdge API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://app.binaryedge.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'C99 API Key': {'description': 'Enter your api key under your profile after'
-                                                 ' signing up on https://c99.nl.',
-                                  'type': 'String',
-                                  'value': 'None',
-                                  'global': True,
-                                  'default': 'None'},
-                  'Censys API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://censys.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Chaos API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://chaos.projectdiscovery.io.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'Cloudflare API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://cloudflare.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'DNSDB API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://dnsdb.info.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'GitHub API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://github.com.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Hunter API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://hunter.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'IPInfo Access Token': {'description': 'Enter your api key under your profile after'
-                                                         ' signing up on https://ipinfo.io.',
-                                          'type': 'String',
-                                          'value': 'None',
-                                          'global': True,
-                                          'default': 'None'},
-                  'NetworksDB API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://networksdb.io.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'PassiveTotal API Key': {'description': 'Enter your api key under your profile after'
-                                                          ' signing up on https://passivetotal.com .',
-                                           'type': 'String',
-                                           'value': 'None',
-                                           'global': True,
-                                           'default': 'None'},
-                  'ReconDev API Key': {'description': 'Enter your api key under your profile after'
-                                                      ' signing up on https://recon.dev.',
-                                       'type': 'String',
-                                       'value': 'None',
-                                       'global': True,
-                                       'default': 'None'},
-                  'SecurityTrails API Key': {'description': 'Enter your api key under your profile after'
-                                                            ' signing up on https://securitytrails.com.',
-                                             'type': 'String',
-                                             'value': 'None',
-                                             'global': True,
-                                             'default': 'None'},
-                  'Shodan API Key': {'description': 'Enter your api key under your profile after'
-                                                    ' signing up on https://shodan.io.',
-                                     'type': 'String',
-                                     'value': 'None',
-                                     'global': True,
-                                     'default': 'None'},
-                  'Spyse API Key': {'description': 'Enter your api key under your profile after'
-                                                   ' signing up on https://spyse.com.',
-                                    'type': 'String',
-                                    'value': 'None',
-                                    'global': True,
-                                    'default': 'None'},
-                  'ThreatBook API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://threatbook.cn.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'Umbrella API Key': {'description': 'Enter your api key under your profile after'
-                                                      ' signing up on https://umbrella.cisco.com.',
-                                       'type': 'String',
-                                       'value': 'None',
-                                       'global': True,
-                                       'default': 'None'},
-                  'URLScan API Key': {'description': 'Enter your api key under your profile after'
-                                                     ' signing up on https://urlscan.io.',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'WhoisXMLAPI API Key': {'description': 'Enter your api key under your profile after'
-                                                         ' signing up on https://whoisxmlapi.com.',
-                                          'type': 'String',
-                                          'value': 'None',
-                                          'global': True,
-                                          'default': 'None'},
-                  'ZETAlytics API Key': {'description': 'Enter your api key under your profile after'
-                                                        ' signing up on https://zetalytics.com.',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'ZoomEye API Key': {'description': 'Please Enter the Username and password with a space '
-                                                     'in between',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'FacebookCT API Key': {'description': 'Please Enter the api key and secret with a space '
-                                                        'in between. Obtain them at https://developer.facebook.com',
-                                         'type': 'String',
-                                         'value': 'None',
-                                         'global': True,
-                                         'default': 'None'},
-                  'Twitter API Key': {'description': 'Please Enter the api key and secret with a space '
-                                                     'in between. Obtain them at https://developer.twitter.com',
-                                      'type': 'String',
-                                      'value': 'None',
-                                      'global': True,
-                                      'default': 'None'},
-                  'ReconDev.free API Key': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None',
-                      'global': True,
-                      'default': 'None'},
-                  'ReconDev.paid API Key': {
-                      'description':
-                          'Please Enter the api key under your profile after signing up on https://recon.dev',
-                      'type': 'String',
-                      'value': 'None',
-                      'global': True,
-                      'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        import json
-        from ipaddress import ip_address, IPv4Address, IPv6Address
-        import docker
-        import tempfile
-        from docker.errors import APIError
-
-        return_result = []
-        # Generate Config as a temporary file:
-        with tempfile.TemporaryDirectory() as tempDir:
-            tempPath = Path(tempDir).absolute()
-            config = tempfile.NamedTemporaryFile(mode='w+t', prefix='Amass',
-                                                 suffix='Config',
-                                                 dir=tempPath)
-            config.write("share = true\n")
-            config.write("[scope]\n")
-            config.write("port = 80\n")
-            config.write("port = 443\n")
-            config.write("[data_sources]\n")
-            config.write("minimum_ttl = 1440\n")
-            for parameter in parameters:
-                if parameters[f'{parameter}'] != 'None':
-                    field1 = f"[data_sources.{parameter}]"
-                    field2 = f"[data_sources.{parameter}.Credentials]"
-                    if parameter == "ZoomEye":
-                        username, password = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"{field2}\n")
-                        config.write(f"username = {username}\n")
-                        config.write(f"password = {password}\n")
-                    elif parameter == "FacebookCT":
-                        field3, secret = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.app1\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                        config.write(f"secret = {secret}\n")
-                    elif parameter == "Twitter":
-                        field3, secret = parameters[parameter].split(' ', 1)
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.account1\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                        config.write(f"secret = {secret}\n")
-                    elif parameter == "ReconDev.paid":
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.paid\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                    elif parameter == "ReconDev.free":
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"[data_sources.{parameter}.free\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-                    else:
-                        field3 = parameters[f'{parameter}']
-                        config.write(f"{field1}\n")
-                        config.write(f"{field2}\n")
-                        config.write(f"apikey = \"{field3}\"\n")
-            path_to_config = "/" + Path(config.name).name
-            config.seek(0)
-            for entity in entityJsonList:
-                primary_field = entity[list(entity)[1]].strip()
-                try:
-                    client = docker.from_env()
-                    if entity['Entity Type'] == "Domain":
-                        container = client.containers.run("caffix/amass:latest",
-                                                          f"intel -whois -d {primary_field} -config /.config/amass"
-                                                          f"{path_to_config}",
-                                                          volumes={
-                                                              str(tempPath): {'bind': '/.config/amass',
-                                                                              'mode': 'rw'}},
-                                                          remove=True)
-                    elif entity['Entity Type'] == "IP Address":
-                        try:
-                            ip_address(primary_field)
-                        except ValueError:
-                            return "The Entity Provided isn't a valid IP Address"
-                        container = client.containers.run("caffix/amass:latest",
-                                                          f"intel -addr {primary_field} -config "
-                                                          f"/.config/amass{path_to_config}",
-                                                          volumes={
-                                                              str(tempPath): {'bind': '/.config/amass',
-                                                                              'mode': 'rw'}},
-                                                          remove=True)
-                    elif entity['Entity Type'] == "Autonomous System":
-                        if primary_field.startswith('AS'):
-                            primary_field = primary_field[2:]
-                        container = client.containers.run("caffix/amass:latest",
-                                                          f"intel -asn {primary_field}"
-                                                          f" -config /.config/amass{path_to_config}",
-                                                          volumes={
-                                                              str(tempPath): {'bind': '/.config/amass',
-                                                                              'mode': 'rw'}},
-                                                          remove=True)
-                    textFile = tempPath / 'amass.txt'
-                    textContents = ""
-                    if textFile.exists():
-                        with open(textFile, 'r') as textFileHandler:
-                            textContents = textFileHandler.read()
-
-                    client.close()
-                except (APIError, docker.errors.ContainerError) as error:
-                    return "Something happened to the docker container - Cannot continue: " + str(error)
-                uid = entity['uid']
-                for newDomain in textContents.splitlines():
-                    return_result.append([{'Domain Name': newDomain.strip(),
-                                           'Entity Type': 'Domain'},
-                                          {uid: {'Resolution': 'Amass Intel Scan', 'Notes': ''}}])
-
-            config.close()
-        return return_result

Modules/Amass/requirements.txt

@@ -1 +0,0 @@
-docker

Modules/BigMatch/BigMatch.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BigMatch:
-    name = "BigMatch Search"
-    category = "Secrets & Leaks"
-    description = "Find information about a file using https://bigmatch.rev.ng/static/index.html"
-    originTypes = {"Image", "Document", "Archive"}
-    resultTypes = {'Website'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        from pathlib import Path
-        from playwright.sync_api import sync_playwright, TimeoutError, Error
-        from bs4 import BeautifulSoup
-
-        return_result = []
-
-        url = "https://bigmatch.rev.ng/static/index.html"
-        failString = 'Too many strings in binary?'
-        successString = 'Results:'
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-
-            for entity in entityJsonList:
-                uid = entity['uid']
-                file_path = Path(parameters['Project Files Directory']) / entity["File Path"]
-                file_path = file_path.absolute()
-                if not (file_path.exists() and file_path.is_file()):
-                    continue
-                page.wait_for_timeout(3000)
-
-                for _ in range(3):
-                    try:
-                        page.goto(url, wait_until="networkidle", timeout=10000)
-                        inputLocator = page.locator("input")
-                        inputLocator.set_input_files([str(file_path)])
-                        page.wait_for_timeout(3000)
-                        soup = BeautifulSoup(page.content(), 'lxml')
-                        soupText = soup.get_text()
-                        while (failString not in soupText) and (successString not in soupText):
-                            page.wait_for_timeout(1000)
-                            soup = BeautifulSoup(page.content(), 'lxml')
-                            soupText = soup.get_text()
-                        if failString in soupText:
-                            return []
-                        for link in soup.find_all('a'):
-                            potentialLink = link.get('href', None)
-                            if potentialLink is not None:
-                                if 'github' in potentialLink:
-                                    return_result.append([{'URL': potentialLink, 'Entity Type': 'Website'},
-                                                          {uid: {'Resolution': 'BigMatch Github Link', 'Notes': ''}}])
-
-                        break
-                    except TimeoutError:
-                        pass
-                    except Error:
-                        break
-            page.close()
-            browser.close()
-        return return_result

Modules/BigMatch/requirements.txt

@@ -1,2 +0,0 @@
-bs4
-playwright

Modules/BinaryEdge/BinaryEdgeHost.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BinaryEdgeHost:
-    name = "BinaryEdge Host Query"
-    category = "Network Infrastructure"
-    description = "Get information about a host from BinaryEdge."
-    originTypes = {"IP Address", "IPv6 Address"}
-    resultTypes = {'Port'}
-    parameters = {'BinaryEdge API Key': {'description': "Enter your BinaryEdge API key. Sign up for one at "
-                                                        "https://www.binaryedge.io/",
-                                         'type': 'String',
-                                         'value': '',
-                                         'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import json
-
-        baseURL = 'https://api.binaryedge.io/v2/query/ip/'
-        requestHeaders = {'X-Key': parameters['BinaryEdge API Key'].strip()}
-
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            if entity['Entity Type'] == 'IP Address':
-                primaryField = entity['IP Address']
-            elif entity['Entity Type'] == 'IPv6 Address':
-                primaryField = entity['IPv6 Address']
-            else:
-                continue
-            infoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
-            statusCode = infoRequest.status_code
-
-            if statusCode == 401:
-                return "The BinaryEdge API key provided is not valid."
-            elif statusCode == 403:
-                return "The BinaryEdge API key provided does not have permission to access this resource."
-            elif statusCode != 200:
-                continue
-            requestContent = json.loads(infoRequest.content)
-
-            for event in requestContent['events']:
-                for result in event['results']:
-                    originDetails = result['origin']
-                    targetDetails = result['target']
-                    resultDetails = result['result']
-                    if 'state' not in resultDetails['data']:
-                        # Discard return result if it doesn't actually give us useful info about the state of the port.
-                        # This happens in cases where the API returns stuff like the ciphers used in an SSH service.
-                        #   There seems to always be a result with the simple port info, so we will use that one.
-                        continue
-
-                    returnResults.append([{'Port': targetDetails['ip'] + ':' + str(targetDetails['port']) + ':' +
-                                                   targetDetails['protocol'],
-                                           'State': resultDetails['data']['state']['state'],
-                                           'Banner': resultDetails['data']['service'].get('banner', 'N/A'),
-                                           'Product': resultDetails['data']['service'].get('product', 'Unknown'),
-                                           'Entity Type': 'Port'},
-                                          {uid: {'Resolution': 'BinaryEdge Scan Timestamp: ' + str(originDetails['ts']),
-                                                 'Notes': ''}}])
-
-        return returnResults

Modules/BinaryEdge/requirements.txt

@@ -1 +0,0 @@
-requests

Modules/BreachData/HIBP.xml

@@ -1,34 +0,0 @@
-<HIBP>
-    <Data_Breach>
-        <Attributes>
-            <Attribute default="Breach Name" check="String" primary="True">Breach Name</Attribute>
-            <Attribute default="Breach Title" check="String" primary="False">Breach Title</Attribute>
-            <Attribute default="Breach Domain" check="String" primary="False">Breach Domain</Attribute>
-            <Attribute default="0" check="Numbers" primary="False">Breach Pwn Count</Attribute>
-            <Attribute default="Breach Description" check="String" primary="False">Breach Description</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Sensitive</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Verified</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Fabricated</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Retired</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Spam List</Attribute>
-            <Attribute default="False" check="String" primary="False">Breach Is Malware</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Breach Added Date</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Breach Modified Date</Attribute>
-        </Attributes>
-        <Icon>
-            Default.svg
-        </Icon>
-    </Data_Breach>
-    <Paste_Data_Leak>
-        <Attributes>
-            <Attribute default="Paste Identifier" check="String" primary="True">Paste Identifier</Attribute>
-            <Attribute default="Paste Title" check="String" primary="False">Paste Title</Attribute>
-            <Attribute default="Paste Source" check="String" primary="False">Paste Source</Attribute>
-            <Attribute default="Paste ID" check="String" primary="False">Paste ID</Attribute>
-            <Attribute default="0" check="Numbers" primary="False">Paste Email Count</Attribute>
-        </Attributes>
-        <Icon>
-            Default.svg
-        </Icon>
-    </Paste_Data_Leak>
-</HIBP>

Modules/BreachData/HIBPBreachToDomain.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HIBPBreachToDomain:
-
-    name = "HIBP Breach To Domain"
-    category = "Leaked Data"
-    description = "Get the domain of the primary website that a data breach occurred on."
-    originTypes = {'Data Breach'}
-    resultTypes = {'Domain'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-
-        returnResults = []
-
-        for entity in entityJsonList:
-            domainMaybe = entity.get('Breach Domain')
-            if isinstance(domainMaybe, str):
-                if domainMaybe.strip() != '':
-                    returnResults.append([{'Domain Name': domainMaybe,
-                                           'Entity Type': 'Domain'},
-                                          {entity['uid']: {'Resolution': 'Data Breach to Domain',
-                                                           'Notes': ''}}])
-
-        return returnResults

Modules/BreachData/HaveIBeenPwnedBreachDomains.py

@@ -1,85 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HaveIBeenPwnedBreachDomains:
-    name = "HIBP Breach Domain Lookup"
-    category = "Leaked Data"
-    description = "Find breaches associated with a specified domain."
-    originTypes = {'Domain'}
-    resultTypes = {'Data Breach'}
-    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
-                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
-                                   'type': 'String',
-                                   'value': '',
-                                   'global': True,
-                                   'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import json
-        from time import sleep
-
-        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
-        from PySide6.QtGui import QImage
-
-        baseURL = "https://haveibeenpwned.com/api/v3/breaches?domain="
-        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
-
-        returnResults = []
-
-        count = 0
-        while count < len(entityJsonList):
-            entity = entityJsonList[count]
-            primaryField = entity['Domain Name']
-            breachInfoRequest = requests.get(baseURL + primaryField, headers=requestHeaders)
-            statusCode = breachInfoRequest.status_code
-            if statusCode == 200:
-                breachContent = json.loads(breachInfoRequest.content)
-
-                for breach in breachContent:
-                    try:
-                        breachLogoIconRequest = requests.get(breach['LogoPath'])
-                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
-                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
-                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
-
-                        # Rotate the breach domain logo upside down
-                        breachIconImageRotated = breachIconImageScaled.mirrored()
-
-                        breachIconByteArrayFin = QByteArray()
-                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
-                        breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
-                        breachIconImageRotated.save(breachImageBuffer, "PNG")
-                        breachImageBuffer.close()
-                    except Exception:
-                        breachIconByteArrayFin = None
-
-                    # If Breach Date is None, then default to entity creation date.
-                    returnResults.append([{'Breach Name': breach['Name'],
-                                           'Breach Title': breach['Title'],
-                                           'Breach Domain': breach['Domain'],
-                                           'Breach Pwn Count': str(breach['PwnCount']),
-                                           'Breach Description': breach['Description'],
-                                           'Breach Is Sensitive': str(breach['IsSensitive']),
-                                           'Breach Is Verified': str(breach['IsVerified']),
-                                           'Breach Is Fabricated': str(breach['IsFabricated']),
-                                           'Breach Is Retired': str(breach['IsRetired']),
-                                           'Breach Is Spam List': str(breach['IsSpamList']),
-                                           'Breach Is Malware': str(breach['IsMalware']),
-                                           'Breach Added Date': breach['AddedDate'],
-                                           'Breach Modified Date': breach['ModifiedDate'],
-                                           'Entity Type': 'Data Breach',
-                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
-                                           'Date Created': breach['BreachDate']},
-                                          {entity['uid']: {'Resolution': 'Contained in Breach',
-                                                           'Notes': ''}}])
-            elif statusCode == 401:
-                return "The HIBP API Key provided is invalid."
-            elif statusCode == 429:
-                sleep(2)
-                continue
-            elif statusCode == 503:
-                return "The HIBP Service is unavailable."
-            sleep(1.7)
-            count += 1
-        return returnResults

Modules/BreachData/HaveIBeenPwnedBreaches.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HaveIBeenPwnedBreaches:
-    name = "HIBP Breach Lookup"
-    category = "Leaked Data"
-    description = "Find all breaches that an account has been involved in. Note that Date Created for breaches is an " \
-                  "estimate."
-    originTypes = {'Email Address', 'Phone Number'}
-    resultTypes = {'Data Breach'}
-    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
-                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
-                                   'type': 'String',
-                                   'value': '',
-                                   'global': True,
-                                   'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import json
-        from time import sleep
-        from urllib.parse import quote_plus
-
-        from PySide6.QtCore import QByteArray, QBuffer, QIODevice, QSize
-        from PySide6.QtGui import QImage
-
-        baseURL = "https://haveibeenpwned.com/api/v3/breachedaccount/"
-        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
-
-        returnResults = []
-
-        count = 0
-        while count < len(entityJsonList):
-            entity = entityJsonList[count]
-            primaryField = entity[list(entity)[1]]
-            breachInfoRequest = requests.get(baseURL + quote_plus(primaryField) + '?truncateResponse=false',
-                                             headers=requestHeaders)
-            statusCode = breachInfoRequest.status_code
-            if statusCode == 200:
-                breachContent = json.loads(breachInfoRequest.content)
-
-                for breach in breachContent:
-                    try:
-                        breachLogoIconRequest = requests.get(breach['LogoPath'])
-                        breachIconByteArray = QByteArray(breachLogoIconRequest.content)
-                        breachIconImageOriginal = QImage().fromData(breachIconByteArray)
-                        breachIconImageScaled = breachIconImageOriginal.scaled(QSize(40, 40))
-
-                        # Rotate the breach domain logo upside down
-                        breachIconImageRotated = breachIconImageScaled.mirrored()
-
-                        breachIconByteArrayFin = QByteArray()
-                        breachImageBuffer = QBuffer(breachIconByteArrayFin)
-                        breachImageBuffer.open(QIODevice.OpenModeFlag.WriteOnly)
-                        breachIconImageRotated.save(breachImageBuffer, "PNG")
-                        breachImageBuffer.close()
-                    except Exception:
-                        breachIconByteArrayFin = None
-
-                    # If Breach Date is None, then default to entity creation date.
-                    returnResults.append([{'Breach Name': breach['Name'],
-                                           'Breach Title': breach['Title'],
-                                           'Breach Domain': breach['Domain'],
-                                           'Breach Pwn Count': str(breach['PwnCount']),
-                                           'Breach Description': breach['Description'],
-                                           'Breach Is Sensitive': str(breach['IsSensitive']),
-                                           'Breach Is Verified': str(breach['IsVerified']),
-                                           'Breach Is Fabricated': str(breach['IsFabricated']),
-                                           'Breach Is Retired': str(breach['IsRetired']),
-                                           'Breach Is Spam List': str(breach['IsSpamList']),
-                                           'Breach Is Malware': str(breach['IsMalware']),
-                                           'Breach Added Date': breach['AddedDate'],
-                                           'Breach Modified Date': breach['ModifiedDate'],
-                                           'Entity Type': 'Data Breach',
-                                           'Icon': breachIconByteArrayFin,  # If None -> Default breach icon.
-                                           'Date Created': breach['BreachDate']},
-                                          {entity['uid']: {'Resolution': 'Contained in Breach',
-                                                           'Notes': ''}}])
-            elif statusCode == 401:
-                return "The HIBP API Key provided is invalid."
-            elif statusCode == 429:
-                sleep(2)
-                continue
-            elif statusCode == 503:
-                return "The HIBP Service is unavailable."
-            sleep(1.7)
-            count += 1
-        return returnResults

Modules/BreachData/HaveIBeenPwnedPassword.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HaveIBeenPwnedPassword:
-    name = "HIBP Password Lookup"
-    category = "Leaked Data"
-    description = "Check whether the given password was found in breaches."
-    originTypes = {'Phrase'}
-    resultTypes = {'Phrase'}
-    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
-                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
-                                   'type': 'String',
-                                   'value': '',
-                                   'global': True,
-                                   'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from time import sleep
-        from hashlib import sha1
-
-        baseURL = "https://api.pwnedpasswords.com/range/"
-        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
-
-        returnResults = []
-
-        count = 0
-        while count < len(entityJsonList):
-            entity = entityJsonList[count]
-
-            primaryField = sha1(entity[list(entity)[1]].encode('utf-8')).hexdigest().upper()
-            hashPrefix = primaryField[:5]
-            hashSuffix = primaryField[5:]
-
-            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
-            statusCode = breachInfoRequest.status_code
-            if statusCode == 200:
-                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
-
-                for password in pwnedPasswordContent:
-                    if hashSuffix in password:
-                        returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
-                                                         f"in breach data.",
-                                               'Entity Type': 'Phrase'},
-                                              {entity['uid']: {'Resolution': 'Pwned Password',
-                                                               'Notes': ''}}])
-                        break
-            elif statusCode == 401:
-                return "The HIBP API Key provided is invalid."
-            elif statusCode == 429:
-                sleep(2)
-                continue
-            elif statusCode == 503:
-                return "The HIBP Service is unavailable."
-            sleep(1.7)
-            count += 1
-        return returnResults

Modules/BreachData/HaveIBeenPwnedPasswordHash.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HaveIBeenPwnedPasswordHash:
-    name = "HIBP Password Hash Lookup"
-    category = "Leaked Data"
-    description = "Check whether the given password hash was found in breaches."
-    originTypes = {'Hash', 'Phrase'}
-    resultTypes = {'Phrase'}
-    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
-                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
-                                   'type': 'String',
-                                   'value': '',
-                                   'global': True,
-                                   'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        from time import sleep
-
-        baseURL = "https://api.pwnedpasswords.com/range/"
-        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
-
-        returnResults = []
-
-        count = 0
-        while count < len(entityJsonList):
-            entity = entityJsonList[count]
-
-            primaryField = entity[list(entity)[1]].upper()
-            if len(primaryField) != 40:
-                continue
-            hashPrefix = primaryField[:5]
-            hashSuffix = primaryField[5:]
-
-            breachInfoRequest = requests.get(baseURL + hashPrefix, headers=requestHeaders)
-            statusCode = breachInfoRequest.status_code
-            if statusCode == 200:
-                pwnedPasswordContent = breachInfoRequest.content.decode('utf-8').split('\r\n')
-
-                for password in pwnedPasswordContent:
-                    if hashSuffix in password:
-                        returnResults.append([{'Phrase': f"Password Hash Found {password.split(':')[1]} times "
-                                                         f"in breach data.",
-                                               'Entity Type': 'Phrase'},
-                                              {entity['uid']: {'Resolution': 'Pwned Password',
-                                                               'Notes': ''}}])
-                        break
-            elif statusCode == 401:
-                return "The HIBP API Key provided is invalid."
-            elif statusCode == 429:
-                sleep(2)
-                continue
-            elif statusCode == 503:
-                return "The HIBP Service is unavailable."
-            sleep(1.7)
-            count += 1
-        return returnResults

Modules/BreachData/HaveIBeenPwnedPastes.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-
-
-class HaveIBeenPwnedPastes:
-    name = "HIBP Paste Lookup"
-    category = "Leaked Data"
-    description = "Find all pastes that an account has been involved in."
-    originTypes = {'Email Address'}
-    resultTypes = {'Paste Data Leak'}
-    parameters = {'HIBP API Key': {'description': 'Enter your "Have I Been Pwned" API key. '
-                                                  'You can get a key here: https://haveibeenpwned.com/API/Key',
-                                   'type': 'String',
-                                   'value': '',
-                                   'global': True,
-                                   'default': 'None'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import json
-        from time import sleep
-        from urllib.parse import quote_plus
-
-        baseURL = "https://haveibeenpwned.com/api/v3/pasteaccount/"
-        requestHeaders = {'hibp-api-key': parameters['HIBP API Key'].strip(), 'user-agent': 'LinkScope Client'}
-
-        returnResults = []
-
-        count = 0
-        while count < len(entityJsonList):
-            entity = entityJsonList[count]
-            emailAddress = entity['Email Address']
-            pasteInfoRequest = requests.get(baseURL + quote_plus(emailAddress), headers=requestHeaders)
-            statusCode = pasteInfoRequest.status_code
-            if statusCode == 200:
-                pasteContent = json.loads(pasteInfoRequest.content)
-
-                for paste in pasteContent:
-                    pasteID = paste['Id']
-                    pasteSource = paste['Source']
-
-                    # If Paste Date is None, then default to entity creation date.
-                    returnResults.append([{'Paste Identifier': f'{pasteSource} | {pasteID}',
-                                           'Paste Title': paste['Title'],
-                                           'Paste Source': pasteSource,
-                                           'Paste ID': pasteID,
-                                           'Paste Email Count': str(paste['EmailCount']),
-                                           'Entity Type': 'Paste Data Leak',
-                                           'Date Created': paste['Date']},
-                                          {entity['uid']: {'Resolution': 'Contained in Paste',
-                                                           'Notes': ''}}])
-
-            elif statusCode == 401:
-                return "The HIBP API Key provided is invalid."
-            elif statusCode == 429:
-                sleep(2)
-                continue
-            elif statusCode == 503:
-                return "The HIBP Service is unavailable."
-            sleep(1.7)
-            count += 1
-        return returnResults

Modules/BreachData/requirements.txt

@@ -1 +0,0 @@
-requests

Modules/CryptoCurrency/BlockChainAddressDestinations.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainAddressDestinations:
-
-    name = "Get Outbound Transactions for Bitcoin Address"
-    category = "CryptoCurrency"
-    description = "Returns the Bitcoin transactions where cryptocurrency was sent from this address."
-    originTypes = {'BTC Address'}
-    resultTypes = {'BTC Transaction'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-
-        returnResults = []
-
-        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity['BTC Address']
-
-            try:
-                addressDetails = requests.get(apiEndpointAddress + primaryField).json()
-                if addressDetails.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            blockTransactions = addressDetails.get('txs', [])
-
-            for transaction in blockTransactions:
-                inputValue = 0
-                isInAddr = False
-                for transactionInput in transaction.get('inputs', []):
-                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
-                    if transactionInput['prev_out']['addr'] == primaryField:
-                        isInAddr = True
-
-                if not isInAddr:
-                    continue
-
-                outputValue = 0
-                for transactionOutput in transaction.get('out', []):
-                    outputValue += (int(transactionOutput['value']) / 100000000)
-
-                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
-
-                returnResults.append(
-                    [{'Transaction Hash': transaction['hash'],
-                      'Input Value (BTC)': str(inputValue),
-                      'Output Value (BTC)': str(outputValue),
-                      'Fee': str(transaction['fee']),
-                      'Number of Inputs': str(transaction['vin_sz']),
-                      'Number of Outputs': str(transaction['vout_sz']),
-                      'Transaction Index': str(transaction['tx_index']),
-                      'Size': str(transaction['size']),
-                      'Height': str(transaction['block_height']),
-                      'Entity Type': 'BTC Transaction',
-                      'Date Created': timestamp},
-                     {uid: {'Resolution': 'BTC Transaction',
-                            'Notes': ''}}])
-
-            time.sleep(5)
-
-        return returnResults

Modules/CryptoCurrency/BlockChainAddressSources.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainAddressSources:
-
-    name = "Get Inbound Transactions for Bitcoin Address"
-    category = "CryptoCurrency"
-    description = "Returns the Bitcoin transactions where cryptocurrency was sent to this address."
-    originTypes = {'BTC Address'}
-    resultTypes = {'BTC Transaction'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-
-        returnResults = []
-        returnResultResolutions = {}
-
-        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
-
-        for entity in entityJsonList:
-            primaryField = entity['BTC Address']
-
-            try:
-                addressDetails = requests.get(apiEndpointAddress + primaryField).json()
-                if addressDetails.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            blockTransactions = addressDetails.get('txs', [])
-
-            for transaction in blockTransactions:
-                outputValue = 0
-                isOutAddr = False
-                for transactionOutput in transaction.get('out', []):
-                    outputValue += (int(transactionOutput['value']) / 100000000)
-                    if transactionOutput['addr'] == primaryField:
-                        isOutAddr = True
-
-                if not isOutAddr:
-                    continue
-
-                inputValue = 0
-                for transactionInput in transaction.get('inputs', []):
-                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
-
-                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
-                returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
-
-                returnResults.append(
-                    [{'Transaction Hash': transaction['hash'],
-                      'Input Value (BTC)': str(inputValue),
-                      'Output Value (BTC)': str(outputValue),
-                      'Fee': str(transaction['fee']),
-                      'Number of Inputs': str(transaction['vin_sz']),
-                      'Number of Outputs': str(transaction['vout_sz']),
-                      'Transaction Index': str(transaction['tx_index']),
-                      'Size': str(transaction['size']),
-                      'Height': str(transaction['block_height']),
-                      'Entity Type': 'BTC Transaction',
-                      'Date Created': timestamp},
-                     {'^^^': {'Resolution': 'NULL',
-                              'Notes': ''}}])
-
-            time.sleep(5)
-            returnResults.append([{'BTC Address': primaryField,
-                                   'Entity Type': 'BTC Address'},
-                                  returnResultResolutions])
-            returnResultResolutions = {}
-        return returnResults

Modules/CryptoCurrency/BlockChainBlock.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainBlock:
-
-    name = "Get Bitcoin Block Details"
-    category = "CryptoCurrency"
-    description = "Returns the details of a particular bitcoin block."
-    originTypes = {'Hash', 'Phrase', 'BTC Block'}
-    resultTypes = {'BTC Block'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/rawblock/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity[list(entity)[1]]
-
-            try:
-                details = requests.get(apiEndpoint + primaryField).json()
-                if details.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
-
-            returnResults.append(
-                [{'Block Address': details['hash'],
-                  'Previous Block': details['prev_block'],
-                  'Merkle Root': details['mrkl_root'],
-                  'Relayed By': details['relayed_by'],
-                  'Nonce': str(details['nonce']),
-                  'Bits': str(details['bits']),
-                  'Size': str(details['size']),
-                  'Block Index': str(details['block_index']),
-                  'Height': str(details['height']),
-                  'Main Chain': str(details['main_chain']),
-                  'Entity Type': 'BTC Block',
-                  'Date Created': timestamp},
-                 {uid: {'Resolution': 'Bitcoin Block Details',
-                        'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainBlockHeight.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainBlockHeight:
-
-    name = "Get Bitcoin Blocks At Height"
-    category = "CryptoCurrency"
-    description = "Returns the details of all bitcoin blocks at the specified height."
-    originTypes = {'Phrase'}
-    resultTypes = {'BTC Block'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/block-height/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            try:
-                primaryField = int(entity['Phrase'])
-            except ValueError:
-                continue
-
-            try:
-                heightDetails = requests.get(apiEndpoint + str(primaryField)).json()
-                if heightDetails.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            for details in heightDetails['blocks']:
-                timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
-
-                returnResults.append(
-                    [{'Block Address': details['hash'],
-                      'Previous Block': details['prev_block'],
-                      'Merkle Root': details['mrkl_root'],
-                      'Relayed By': details['relayed_by'],
-                      'Nonce': str(details['nonce']),
-                      'Bits': str(details['bits']),
-                      'Size': str(details['size']),
-                      'Block Index': str(details['block_index']),
-                      'Height': str(details['height']),
-                      'Main Chain': str(details['main_chain']),
-                      'Entity Type': 'BTC Block',
-                      'Date Created': timestamp},
-                     {uid: {'Resolution': 'Bitcoin Block Address',
-                            'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainBlockNext.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainBlockNext:
-
-    name = "Get Next Bitcoin Block"
-    category = "CryptoCurrency"
-    description = "Returns the details of the next bitcoin block."
-    originTypes = {'BTC Block'}
-    resultTypes = {'BTC Block'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/rawblock/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity['Block Address']
-
-            try:
-                currDetails = requests.get(apiEndpoint + primaryField).json()
-                if currDetails.get('error') is not None:
-                    continue
-                nextBlockHashList = currDetails.get('next_block')
-                # Ignore nonexistent or indeterminate 'next' blocks.
-                if nextBlockHashList is None or len(nextBlockHashList) > 1:
-                    continue
-                time.sleep(5)
-                details = requests.get(apiEndpoint + nextBlockHashList[0]).json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
-
-            returnResults.append(
-                [{'Block Address': details['hash'],
-                  'Previous Block': details['prev_block'],
-                  'Merkle Root': details['mrkl_root'],
-                  'Relayed By': details['relayed_by'],
-                  'Nonce': str(details['nonce']),
-                  'Bits': str(details['bits']),
-                  'Size': str(details['size']),
-                  'Block Index': str(details['block_index']),
-                  'Height': str(details['height']),
-                  'Main Chain': str(details['main_chain']),
-                  'Entity Type': 'BTC Block',
-                  'Date Created': timestamp},
-                 {uid: {'Resolution': 'Next BTC Block',
-                        'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainBlockPrev.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainBlockPrev:
-
-    name = "Get Previous Bitcoin Block"
-    category = "CryptoCurrency"
-    description = "Returns the details of the previous bitcoin block."
-    originTypes = {'BTC Block'}
-    resultTypes = {'BTC Block'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/rawblock/'
-
-        for entity in entityJsonList:
-            primaryField = entity['Block Address']
-
-            try:
-                currDetails = requests.get(apiEndpoint + primaryField).json()
-                if currDetails.get('error') is not None:
-                    continue
-                prevBlockHash = currDetails.get('prev_block')
-                # Ignore first block.
-                if prevBlockHash == "0000000000000000000000000000000000000000000000000000000000000000":
-                    continue
-                time.sleep(5)
-                details = requests.get(apiEndpoint + prevBlockHash).json()
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            timestamp = datetime.utcfromtimestamp(details.get('time')).isoformat()
-
-            returnResults.append(
-                [{'Block Address': details['hash'],
-                  'Previous Block': details['prev_block'],
-                  'Merkle Root': details['mrkl_root'],
-                  'Relayed By': details['relayed_by'],
-                  'Nonce': str(details['nonce']),
-                  'Bits': str(details['bits']),
-                  'Size': str(details['size']),
-                  'Block Index': str(details['block_index']),
-                  'Height': str(details['height']),
-                  'Main Chain': str(details['main_chain']),
-                  'Entity Type': 'BTC Block',
-                  'Date Created': timestamp},
-                 {'^^^': {'Resolution': 'NULL'}}])
-
-            returnResults.append([{'Block Address': primaryField,
-                                   'Entity Type': 'BTC Block'},
-                                  {len(returnResults) - 1: {'Resolution': 'Next BTC Block'}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainBlockTransactions.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainBlockTransactions:
-
-    name = "Get Bitcoin Block Transactions"
-    category = "CryptoCurrency"
-    description = "Returns the transactions that happened in a particular bitcoin block."
-    originTypes = {'BTC Block'}
-    resultTypes = {'BTC Transaction'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/rawblock/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity['Block Address']
-
-            try:
-                details = requests.get(apiEndpoint + primaryField).json()
-                if details.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            blockTransactions = details.get('tx', [])
-
-            for transaction in blockTransactions:
-                inputValue = 0
-                for transactionInput in transaction.get('inputs', []):
-                    inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
-
-                outputValue = 0
-                for transactionOutput in transaction.get('out', []):
-                    outputValue += (int(transactionOutput['value']) / 100000000)
-
-                timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
-
-                returnResults.append(
-                    [{'Transaction Hash': transaction['hash'],
-                      'Input Value (BTC)': str(inputValue),
-                      'Output Value (BTC)': str(outputValue),
-                      'Fee': str(transaction['fee']),
-                      'Number of Inputs': str(transaction['vin_sz']),
-                      'Number of Outputs': str(transaction['vout_sz']),
-                      'Transaction Index': str(transaction['tx_index']),
-                      'Size': str(transaction['size']),
-                      'Height': str(transaction['block_height']),
-                      'Entity Type': 'BTC Transaction',
-                      'Date Created': timestamp},
-                     {uid: {'Resolution': 'Bitcoin Block Address',
-                            'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainTransaction.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainTransaction:
-
-    name = "Get Bitcoin Transaction"
-    category = "CryptoCurrency"
-    description = "Returns the details of the specified Bitcoin transaction."
-    originTypes = {'BTC Transaction', 'Hash', 'Phrase'}
-    resultTypes = {'BTC Transaction'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        from datetime import datetime
-        returnResults = []
-
-        apiEndpoint = 'https://blockchain.info/rawtx/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity[list(entity)[1]]
-
-            try:
-                transaction = requests.get(apiEndpoint + primaryField).json()
-                if transaction.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            timestamp = datetime.utcfromtimestamp(transaction.get('time')).isoformat()
-
-            inputValue = 0
-            for transactionInput in transaction.get('inputs', []):
-                inputValue += (int(transactionInput['prev_out']['value']) / 100000000)
-
-            outputValue = 0
-            for transactionOutput in transaction.get('out', []):
-                outputValue += (int(transactionOutput['value']) / 100000000)
-
-            returnResults.append(
-                [{'Transaction Hash': transaction['hash'],
-                  'Input Value (BTC)': str(inputValue),
-                  'Output Value (BTC)': str(outputValue),
-                  'Fee': str(transaction['fee']),
-                  'Number of Inputs': str(transaction['vin_sz']),
-                  'Number of Outputs': str(transaction['vout_sz']),
-                  'Transaction Index': str(transaction['tx_index']),
-                  'Size': str(transaction['size']),
-                  'Height': str(transaction['block_height']),
-                  'Entity Type': 'BTC Transaction',
-                  'Date Created': timestamp},
-                 {uid: {'Resolution': 'Bitcoin Transaction Information',
-                        'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainTransactionDestinations.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainTransactionDestinations:
-
-    name = "Get Bitcoin Transaction Destinations"
-    category = "CryptoCurrency"
-    description = "Returns the Bitcoin addresses that received cryptocurrency in the specified transaction."
-    originTypes = {'BTC Transaction'}
-    resultTypes = {'BTC Address'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        returnResults = []
-
-        apiEndpointTransaction = 'https://blockchain.info/rawtx/'
-        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primaryField = entity['Transaction Hash']
-
-            try:
-                transaction = requests.get(apiEndpointTransaction + primaryField).json()
-                if transaction.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            for transactionInput in transaction.get('out', []):
-                time.sleep(5)
-                inputAddress = transactionInput['addr']
-                try:
-                    details = requests.get(apiEndpointAddress + inputAddress).json()
-                    if details.get('error') is not None:
-                        continue
-                except requests.exceptions.ConnectionError:
-                    return "Please check your internet connection"
-
-                returnResults.append(
-                    [{'BTC Address': details['address'],
-                      'Total Transactions': str(details['n_tx']),
-                      'Unredeemed Transactions': str(details['n_unredeemed']),
-                      'Total BTC Received': str(details['total_received'] / 100000000),
-                      'Total BTC Sent': str(details['total_sent'] / 100000000),
-                      'Current Balance': str(details['final_balance'] / 100000000),
-                      'Entity Type': 'BTC Address'},
-                     {uid: {'Resolution': 'Bitcoin Transaction',
-                            'Notes': ''}}])
-
-            time.sleep(5)
-        return returnResults

Modules/CryptoCurrency/BlockChainTransactionSources.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-
-
-class BlockChainTransactionSources:
-
-    name = "Get Bitcoin Transaction Sources"
-    category = "CryptoCurrency"
-    description = "Returns the Bitcoin addresses that sent cryptocurrency in the specified transaction."
-    originTypes = {'BTC Transaction'}
-    resultTypes = {'BTC Address'}
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        returnResults = []
-        returnResultResolutions = {}
-
-        apiEndpointTransaction = 'https://blockchain.info/rawtx/'
-        apiEndpointAddress = 'https://blockchain.info/rawaddr/'
-
-        for entity in entityJsonList:
-            primaryField = entity['Transaction Hash']
-
-            try:
-                transaction = requests.get(apiEndpointTransaction + primaryField).json()
-                if transaction.get('error') is not None:
-                    continue
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-
-            for transactionInput in transaction.get('inputs', []):
-                time.sleep(5)
-                inputAddress = transactionInput['prev_out']['addr']
-                try:
-                    details = requests.get(apiEndpointAddress + inputAddress).json()
-                    if details.get('error') is not None:
-                        continue
-                except requests.exceptions.ConnectionError:
-                    return "Please check your internet connection"
-
-                returnResultResolutions[len(returnResults)] = {'Resolution': 'BTC Transaction'}
-                returnResults.append(
-                    [{'BTC Address': details['address'],
-                      'Total Transactions': str(details['n_tx']),
-                      'Unredeemed Transactions': str(details['n_unredeemed']),
-                      'Total BTC Received': str(details['total_received'] / 100000000),
-                      'Total BTC Sent': str(details['total_sent'] / 100000000),
-                      'Current Balance': str(details['final_balance'] / 100000000),
-                      'Entity Type': 'BTC Address'},
-                     {'^^^': {'Resolution': 'NULL',
-                              'Notes': ''}}])
-
-            time.sleep(5)
-            # Re-add the source entity so that we can point to it.
-            # Only include the primary field, in case the rest of the fields were updated in the meantime.
-            returnResults.append([{'Transaction Hash': primaryField,
-                                   'Entity Type': 'BTC Transaction'},
-                                  returnResultResolutions])
-            returnResultResolutions = {}
-        return returnResults

Modules/CryptoCurrency/CryptoCurrency.xml

@@ -1,26 +0,0 @@
-<CryptoCurrency>
-    <BTC_Block>
-        <Attributes>
-          <Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Block Address</Attribute>
-        </Attributes>
-        <Icon>
-          Bitcoin.svg
-        </Icon>
-    </BTC_Block>
-    <BTC_Transaction>
-        <Attributes>
-            <Attribute default="0000000000000000000000000000000000000000000000000000000000000000" check="String" primary="True">Transaction Hash</Attribute>
-        </Attributes>
-        <Icon>
-            Transaction.svg
-        </Icon>
-    </BTC_Transaction>
-    <BTC_Address>
-        <Attributes>
-            <Attribute default="1AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" check="String" primary="True">BTC Address</Attribute>
-        </Attributes>
-        <Icon>
-            CryptoWallet.svg
-        </Icon>
-    </BTC_Address>
-</CryptoCurrency>

Modules/CryptoCurrency/EtherScanGetBalance.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EtherScanGetBalance:
-    name = "EtherScan.io Get Balance"
-    category = "CryptoCurrency"
-    description = "EtherScan get the balance of the selected account"
-    originTypes = {"Crypto Wallet"}
-    resultTypes = {'Crypto Wallet'}
-    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io.",
-                                        'type': 'String',
-                                        'value': '',
-                                        'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        return_result = []
-
-        api_key = parameters['EtherScan API Key']
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primary_field = entity['Wallet Address']
-            crafted_url = f"https://api.etherscan.io/api?module=account&action=balance" \
-                          f"&address={primary_field}&tag=latest&apikey={api_key}"
-            try:
-                response = requests.get(crafted_url)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            response = response.json()
-            return_result.append([{'Amount': response['result'],
-                                   'Currency': 'Ethereum',
-                                   'Entity Type': 'Currency'},
-                                  {uid: {'Resolution': 'EtherScan.io Account Balance', 'Notes': ''}}])
-            time.sleep(0.2)
-
-        return return_result

Modules/CryptoCurrency/EtherScanGetBlocksMined.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EtherScanGetBlocksMined:
-    name = "EtherScan.io Get Blocks Mined"
-    category = "CryptoCurrency"
-    description = "EtherScan Blocks Mined from the selected account"
-    originTypes = {"Crypto Wallet"}
-    resultTypes = {'Crypto Wallet'}
-    parameters = {'EtherScan API Key': {'description': "Enter the api key under your profile after signing up at "
-                                                       "https://etherscan.io.",
-                                        'type': 'String',
-                                        'value': '',
-                  'global': True}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        return_result = []
-
-        api_key = parameters['EtherScan API Key']
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            primary_field = entity['Wallet Address']
-            crafted_url = f"https://api.etherscan.io/api?module=account&action=getminedblocks" \
-                          f"&address={primary_field}&tag=latest&apikey={api_key}"
-            try:
-                response = requests.get(crafted_url)
-            except requests.exceptions.ConnectionError:
-                return "Please check your internet connection"
-            response = response.json()
-            return_result.append([{'Phrase': response['result'],
-                                   'Entity Type': 'Phrase'},
-                                  {uid: {'Resolution': 'EtherScan.io Blocks Mined', 'Notes': ''}}])
-            time.sleep(0.2)
-
-        return return_result

Modules/CryptoCurrency/ToCryptoWallet.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-
-
-class ToCryptoWallet:
-    name = "BTC Address To Crypto Wallet"
-    category = "CryptoCurrency"
-    description = "Convert BTC Address entities to Crypto Wallet entities."
-    originTypes = {'BTC Address'}
-    resultTypes = {'Crypto Wallet'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-
-        returnResults = []
-
-        for entity in entityJsonList:
-            primaryField = entity['BTC Address']
-            returnResults.append([{'Wallet Address': primaryField,
-                                   'Currency Name': 'Bitcoin',
-                                   'Entity Type': 'Crypto Wallet'},
-                                  {entity['uid']: {'Resolution': 'To Crypto Wallet',
-                                                   'Notes': ''}}])
-
-        return returnResults

Modules/CryptoCurrency/requirements.txt

@@ -1,2 +0,0 @@
-requests
-datetime

Modules/EDGAR_&_EFD/CompanyInfo.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-
-class CompanyInfo:
-    # A string that is treated as the name of this resolution.
-    name = "Get Company Info"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes containing Company Information"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Phrase, SIC, EIN, Address'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(1)
-            search_url = f'https://data.sec.gov/submissions/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            time.sleep(1)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-
-            exchanges = data['exchanges']
-            for exchange in exchanges:
-                returnResults.append([{'Exchange Name': exchange,
-                                       'Entity Type': 'Exchange'},
-                                      {uid: {'Resolution': 'Exchange',
-                                             'Notes': ''}}])
-
-            tickers = data['tickers']
-            for ticker in tickers:
-                returnResults.append([{'Ticker ID': ticker,
-                                       'Entity Type': 'Ticker'},
-                                      {uid: {'Resolution': 'Ticker',
-                                             'Notes': ''}}])
-
-            if data['insiderTransactionForOwnerExists'] == 1:
-                returnResults.append([{'Phrase': 'Insider Transaction For Owner Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-            else:
-                returnResults.append([{'Phrase': 'Insider Transaction For Owner Does Not Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-
-            if data['insiderTransactionForIssuerExists'] == 1:
-                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-            else:
-                returnResults.append([{'Phrase': 'Insider Transaction For Issuer Does Not Exists',
-                                       'Entity Type': 'Phrase'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-
-            if data['sic'] is not None:
-                returnResults.append([{'SIC': str(data['sic']),
-                                       'Description': data['sicDescription'],
-                                       'Entity Type': 'SIC'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-            if data['ein'] is not None:
-                returnResults.append([{'EIN': str(data['ein']),
-                                       'Entity Type': 'EIN'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-            if data['addresses'] is not None:
-                returnResults.append([{'Street Address': data['addresses']['mailing']['street1'],
-                                       'Postal Code': data['addresses']['mailing']['zipCode'],
-                                       'Country': data['addresses']['mailing']['stateOrCountry'],
-                                       'Locality': data['addresses']['mailing']['city'],
-                                       'Entity Type': 'Address'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-
-            if data['addresses']['mailing']['street1'] != data['addresses']['business']['street1']:
-                returnResults.append([{'Street Address': data['addresses']['business']['street1'],
-                                       'Postal Code': data['addresses']['business']['zipCode'],
-                                       'Country': data['addresses']['business']['stateOrCountry'],
-                                       'Locality': data['addresses']['business']['city'],
-                                       'Entity Type': 'Address'},
-                                      {uid: {'Resolution': '',
-                                             'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/CompanyToCIK.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-
-class CompanyToCIK:
-    # A string that is treated as the name of this resolution.
-    name = "Get CIK ID From Company"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes of contact info for websites"
-
-    originTypes = {'Phrase', 'Company'}
-
-    resultTypes = {'Phrase'}
-
-    parameters = {}
-
-    def resolution(self, entityJsonList, parameters):
-        from bs4 import BeautifulSoup
-        from playwright.sync_api import sync_playwright, TimeoutError, Error
-
-        returnResults = []
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080}
-            )
-            page = context.new_page()
-            for entity in entityJsonList:
-                page.wait_for_timeout(1000)
-                uid = entity['uid']
-                search_term = entity[list(entity)[1]]
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(f'https://www.sec.gov/cgi-bin/browse-edgar?company={search_term}',
-                                  wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                    except Error:
-                        break
-                if not pageResolved:
-                    continue
-
-                soup = BeautifulSoup(page.content(), 'lxml')
-
-                count = 0
-                temp = None
-                for td_element in soup.find_all('td'):
-                    if td_element.text:
-                        text = td_element.text
-                        splitText = text.split('SIC')[0]
-                        if count == 0:
-                            temp = [{'CIK': splitText,
-                                     'Entity Type': 'Edgar ID'},
-                                    {len(returnResults): {'Resolution': 'CIK Edgar ID',
-                                                          'Notes': ''}}]
-                        elif count == 1:
-                            returnResults.append([{'Company Name': splitText,
-                                                   'Entity Type': 'Company'},
-                                                  {uid: {'Resolution': 'Edgar Company',
-                                                         'Notes': ''}}])
-                            returnResults.append(temp)
-                        elif count == 2:
-                            returnResults.append([{'Phrase': "State: " + splitText,
-                                                   'Entity Type': 'Phrase'},
-                                                  {len(returnResults) - 1: {'Resolution': 'Edgar Company State',
-                                                                            'Notes': ''}}])
-                        count = (count + 1) % 3
-
-            page.close()
-            browser.close()
-        return returnResults

Modules/EDGAR_&_EFD/EFDByFromDate.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EFDByFromDate:
-    name = 'Get EFD Reports From Date'
-    category = "US Senate Financial Info"
-    description = 'Get EFD reports starting from the date specified by the input entities.'
-    originTypes = {'Date'}
-    resultTypes = {'Politically Exposed Person', 'Website'}
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'},
-                  'To Date': {'description': 'Records will be collected from the Start Date provided by the input '
-                                             'entities. NOTE: The Start Date is assumed to be in ISO format.\n'
-                                             'An End Date is required to complete the Date constraints. '
-                                             'Please input the End Date for the search in the format mm/dd/yyyy',
-                              'type': 'String',
-                              'value': ''},
-                  'Filer Type': {'description': 'Please select the Office you wish to search records for.',
-                                 'type': 'MultiChoice',
-                                 'value': {'Senator',
-                                           'Candidate',
-                                           'Former Senator',
-                                           }},
-                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
-                                  'type': 'MultiChoice',
-                                  'value': {'Annual',
-                                            'Periodic Transactions',
-                                            'Due Date Extension',
-                                            'Blind Trusts',
-                                            'Other Documents',
-                                            }}}
-
-    def resolution(self, entityJsonList, parameters):
-        from datetime import datetime
-        from playwright.sync_api import sync_playwright, TimeoutError, Error
-        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
-
-        returnResults = []
-
-        try:
-            maxResults = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter."
-
-        if maxResults <= 0:
-            return []
-
-        try:
-            toDate = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
-        except ValueError:
-            return "Invalid End Date specified."
-
-        url = 'https://efdsearch.senate.gov/search/'
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-
-            pageResolved = False
-            for _ in range(5):
-                try:
-                    page.goto(url, wait_until="networkidle", timeout=10000)
-                    pageResolved = True
-                    break
-                except TimeoutError:
-                    pass
-                except Error:
-                    break
-            if not pageResolved:
-                return "Could not access EFD Search website."
-
-            try:
-                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
-            except TimeoutError:
-                return "The EFD search website is unresponsive."
-            except Error:
-                return "Connection Error."
-            page.wait_for_timeout(1000)
-
-            for entity in entityJsonList:
-                try:
-                    # Assume ISO format - guessing
-                    date = datetime.fromisoformat(entity['Date'])
-                except ValueError:
-                    continue
-                if toDate < date:
-                    continue
-                date = date.strftime('%m/%d/%Y')
-                toDate = toDate.strftime('%m/%d/%Y')
-                uid = entity['uid']
-                page.wait_for_timeout(1000)
-
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(url, wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                    except Error:
-                        break
-                if not pageResolved:
-                    continue
-
-                try:
-                    page.fill("input[name=\"submitted_end_date\"]", toDate)
-                    page.fill("input[name=\"submitted_start_date\"]", date)
-                    if 'Senator' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Senator\")")
-                    if 'Candidate' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Candidate\")")
-                    if 'Former Senator' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Former Senator\")")
-
-                    if 'Annual' in parameters['Report Type']:
-                        page.click("text=Annual")
-                    if 'Periodic Transactions' in parameters['Report Type']:
-                        page.click("text=Periodic Transactions")
-                    if 'Due Date Extension' in parameters['Report Type']:
-                        page.click("text=Due Date Extension")
-                    if 'Blind Trusts' in parameters['Report Type']:
-                        page.click("text=Blind Trusts")
-                    if 'Other Documents' in parameters['Report Type']:
-                        page.click("text=Other Documents")
-
-                    page.click("text=Search Reports")
-
-                    entriesInfo = page.locator('#filedReports_info')
-                    entriesInfo.wait_for(state='visible')
-                    currentFirstIndex = 1
-                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-                    resultCount = 0
-
-                    if lastIndex == 0:
-                        continue
-
-                    # Need to click twice to sort by most recent.
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-
-                    while True:
-                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
-                        for record in soup:
-                            if isinstance(record, Tag) and record.get('class'):
-                                recordFields = record.childGenerator()
-                                senateName = next(recordFields).text
-                                senateName += " " + next(recordFields).text
-                                office = next(recordFields).text
-                                report = next(recordFields)
-                                reportType = report.text
-                                reportLink = next(report.children).get('href')
-                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
-                                resultCount += 1
-                                childIndex = len(returnResults)
-                                returnResults.append([{'Full Name': senateName,
-                                                       'Office': office,
-                                                       'Entity Type': 'Politically Exposed Person'},
-                                                      {uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
-                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
-                                                       'Report Type': reportType,
-                                                       'Entity Type': 'Website'},
-                                                      {childIndex: {'Resolution': 'Filed Disclosure Report',
-                                                                    'Notes': '',
-                                                                    'Date Created': dateCreated}}])
-                                if resultCount == maxResults:
-                                    break
-
-                        # Break if we've read enough records, or we ran out of records on this page.
-                        if resultCount == maxResults or currentLastIndex == lastIndex:
-                            break
-
-                        # We've read all the available records, so we click next.
-                        page.click("text=Next")
-                        entriesInfo.wait_for(state='visible')
-                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
-                            page.wait_for_timeout(1000)
-                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
-                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-
-                except TimeoutError:
-                    continue
-                except Exception as e:
-                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
-
-            page.close()
-            browser.close()
-        return returnResults

Modules/EDGAR_&_EFD/EFDByName.py

@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EFDByName:
-    name = 'Get EFD Reports by Name'
-    category = "US Senate Financial Info"
-    description = 'Get EFD reports concerning the people specified by the input entities.'
-    originTypes = {'Person', 'Politically Exposed Person'}
-    resultTypes = {'Website'}
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'},
-                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
-                                  'type': 'MultiChoice',
-                                  'value': {'Annual',
-                                            'Periodic Transactions',
-                                            'Due Date Extension',
-                                            'Blind Trusts',
-                                            'Other Documents',
-                                            }}}
-
-    def resolution(self, entityJsonList, parameters):
-        from datetime import datetime
-        from playwright.sync_api import sync_playwright, TimeoutError
-        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
-
-        returnResults = []
-
-        try:
-            maxResults = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter."
-
-        if maxResults <= 0:
-            return []
-
-        url = 'https://efdsearch.senate.gov/search/'
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-
-            pageResolved = False
-            for _ in range(5):
-                try:
-                    page.goto(url, wait_until="networkidle", timeout=10000)
-                    pageResolved = True
-                    break
-                except TimeoutError:
-                    pass
-            if not pageResolved:
-                return "Could not access efdsearch website."
-
-            try:
-                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
-            except TimeoutError:
-                return "The efdsearch website is unresponsive"
-            page.wait_for_timeout(1000)
-
-            for entity in entityJsonList:
-                lastName = entity['Full Name'].split(' ')[-1]
-                firstName = " ".join(entity['Full Name'].split(' ')[:-1])
-
-                uid = entity['uid']
-                page.wait_for_timeout(1000)
-
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(url, wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                if not pageResolved:
-                    continue
-
-                try:
-                    personOccupation = entity['Occupation'].lower()
-                    if personOccupation == 'senator':
-                        page.click("label:has-text(\"Senator\")")
-                    elif personOccupation == 'candidate':
-                        page.click("label:has-text(\"Candidate\")")
-                    elif personOccupation == 'former senator':
-                        page.click("label:has-text(\"Former Senator\")")
-
-                    if 'Annual' in parameters['Report Type']:
-                        page.click("text=Annual")
-                    if 'Periodic Transactions' in parameters['Report Type']:
-                        page.click("text=Periodic Transactions")
-                    if 'Due Date Extension' in parameters['Report Type']:
-                        page.click("text=Due Date Extension")
-                    if 'Blind Trusts' in parameters['Report Type']:
-                        page.click("text=Blind Trusts")
-                    if 'Other Documents' in parameters['Report Type']:
-                        page.click("text=Other Documents")
-
-                    page.fill("[placeholder=\"First name (starts with)\"]", firstName)
-                    page.fill("[placeholder=\"Last name (starts with)\"]", lastName)
-
-                    page.click("text=Search Reports")
-
-                    entriesInfo = page.locator('#filedReports_info')
-                    entriesInfo.wait_for(state='visible')
-                    currentFirstIndex = 1
-                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-                    resultCount = 0
-
-                    if lastIndex == 0:
-                        continue
-
-                    # Need to click twice to sort by most recent.
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-
-                    while True:
-                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
-                        for record in soup:
-                            if isinstance(record, Tag) and record.get('class'):
-                                recordFields = record.childGenerator()
-                                senateName = next(recordFields).text
-                                senateName += " " + next(recordFields).text
-                                office = next(recordFields).text
-                                report = next(recordFields)
-                                reportType = report.text
-                                reportLink = next(report.children).get('href')
-                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
-                                resultCount += 1
-                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
-                                                       'Report Type': reportType,
-                                                       'Entity Type': 'Website'},
-                                                      {uid: {'Resolution': 'Filed Disclosure Report',
-                                                             'Notes': '',
-                                                             'Date Created': dateCreated}}])
-                                if resultCount == maxResults:
-                                    break
-
-                        # Break if we've read enough records, or we ran out of records on this page.
-                        if resultCount == maxResults or currentLastIndex == lastIndex:
-                            break
-
-                        # We've read all the available records, so we click next.
-                        page.click("text=Next")
-                        entriesInfo.wait_for(state='visible')
-                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
-                            page.wait_for_timeout(1000)
-                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
-                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-
-                except TimeoutError:
-                    continue
-                except Exception as e:
-                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
-
-            page.close()
-            browser.close()
-        return returnResults

Modules/EDGAR_&_EFD/EFDByToDate.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-
-
-class EFDByToDate:
-    name = 'Get EFD Reports To Date'
-    category = "US Senate Financial Info"
-    description = 'Get EFD reports ending at the date specified by the input entities.'
-    originTypes = {'Date'}
-    resultTypes = {'Politically Exposed Person', 'Website'}
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return. '
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'},
-                  'To Date': {'description': 'Records will be collected from the End Date provided by the input '
-                                             'entities. NOTE: The End Date is assumed to be in ISO format.\n'
-                                             'A Start Date is required to complete the Date constraints. '
-                                             'Please input the Start Date for the search in the format mm/dd/yyyy',
-                              'type': 'String',
-                              'value': ''},
-                  'Filer Type': {'description': 'Please select the Office you wish to search records for.',
-                                 'type': 'MultiChoice',
-                                 'value': {'Senator',
-                                           'Candidate',
-                                           'Former Senator',
-                                           }},
-                  'Report Type': {'description': 'Please select the Report Type you want to search for.',
-                                  'type': 'MultiChoice',
-                                  'value': {'Annual',
-                                            'Periodic Transactions',
-                                            'Due Date Extension',
-                                            'Blind Trusts',
-                                            'Other Documents',
-                                            }}}
-
-    def resolution(self, entityJsonList, parameters):
-        from datetime import datetime
-        from playwright.sync_api import sync_playwright, TimeoutError, Error
-        from bs4 import BeautifulSoup, SoupStrainer, Doctype, Tag
-
-        returnResults = []
-
-        try:
-            maxResults = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter."
-
-        if maxResults <= 0:
-            return []
-
-        try:
-            date = datetime.strptime(parameters['To Date'], '%m/%d/%Y')
-        except ValueError:
-            return "Invalid End Date specified."
-
-        url = 'https://efdsearch.senate.gov/search/'
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-
-            pageResolved = False
-            for _ in range(5):
-                try:
-                    page.goto(url, wait_until="networkidle", timeout=10000)
-                    pageResolved = True
-                    break
-                except TimeoutError:
-                    pass
-                except Error:
-                    break
-            if not pageResolved:
-                return "Could not access EFD Search website."
-
-            try:
-                page.click("text=I understand the prohibitions on obtaining and use of financial disclosure repor")
-            except TimeoutError:
-                return "The EFD search website is unresponsive."
-            except Error:
-                return "Connection Error."
-            page.wait_for_timeout(1000)
-
-            for entity in entityJsonList:
-                try:
-                    # Assume ISO format - guessing
-                    toDate = datetime.fromisoformat(entity['Date'])
-                except ValueError:
-                    continue
-                if toDate < date:
-                    continue
-                date = date.strftime('%m/%d/%Y')
-                toDate = toDate.strftime('%m/%d/%Y')
-                uid = entity['uid']
-                page.wait_for_timeout(1000)
-
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(url, wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                    except Error:
-                        break
-                if not pageResolved:
-                    continue
-
-                try:
-                    page.fill("input[name=\"submitted_end_date\"]", toDate)
-                    page.fill("input[name=\"submitted_start_date\"]", date)
-                    if 'Senator' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Senator\")")
-                    if 'Candidate' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Candidate\")")
-                    if 'Former Senator' in parameters['Filer Type']:
-                        page.click("label:has-text(\"Former Senator\")")
-
-                    if 'Annual' in parameters['Report Type']:
-                        page.click("text=Annual")
-                    if 'Periodic Transactions' in parameters['Report Type']:
-                        page.click("text=Periodic Transactions")
-                    if 'Due Date Extension' in parameters['Report Type']:
-                        page.click("text=Due Date Extension")
-                    if 'Blind Trusts' in parameters['Report Type']:
-                        page.click("text=Blind Trusts")
-                    if 'Other Documents' in parameters['Report Type']:
-                        page.click("text=Other Documents")
-
-                    page.click("text=Search Reports")
-
-                    entriesInfo = page.locator('#filedReports_info')
-                    entriesInfo.wait_for(state='visible')
-                    currentFirstIndex = 1
-                    currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                    lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-                    resultCount = 0
-
-                    if lastIndex == 0:
-                        continue
-
-                    # Need to click twice to sort by most recent.
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-                    page.click("text=Date Received/Filed")
-                    page.wait_for_timeout(500)
-
-                    while True:
-                        soup = BeautifulSoup(page.content(), 'lxml', parse_only=SoupStrainer('tr'))
-                        for record in soup:
-                            if isinstance(record, Tag) and record.get('class'):
-                                recordFields = record.childGenerator()
-                                senateName = next(recordFields).text
-                                senateName += " " + next(recordFields).text
-                                office = next(recordFields).text
-                                report = next(recordFields)
-                                reportType = report.text
-                                reportLink = next(report.children).get('href')
-                                dateCreated = datetime.strptime(next(recordFields).text, '%m/%d/%Y').isoformat()
-                                resultCount += 1
-                                childIndex = len(returnResults)
-                                returnResults.append([{'Full Name': senateName,
-                                                       'Office': office,
-                                                       'Entity Type': 'Politically Exposed Person'},
-                                                      {uid: {'Resolution': 'EFD Reports', 'Notes': ''}}])
-                                returnResults.append([{'URL': 'https://efdsearch.senate.gov' + reportLink,
-                                                       'Report Type': reportType,
-                                                       'Entity Type': 'Website'},
-                                                      {childIndex: {'Resolution': 'Filed Disclosure Report',
-                                                                    'Notes': '',
-                                                                    'Date Created': dateCreated}}])
-                                if resultCount == maxResults:
-                                    break
-
-                        # Break if we've read enough records, or we ran out of records on this page.
-                        if resultCount == maxResults or currentLastIndex == lastIndex:
-                            break
-
-                        # We've read all the available records, so we click next.
-                        page.click("text=Next")
-                        entriesInfo.wait_for(state='visible')
-                        while currentFirstIndex == int(entriesInfo.inner_text().split(" ")[1]):
-                            page.wait_for_timeout(1000)
-                        currentFirstIndex = int(entriesInfo.inner_text().split(" ")[1])
-                        currentLastIndex = int(entriesInfo.inner_text().split(" ")[3])
-                        lastIndex = int(entriesInfo.inner_text().split(" ")[5])
-
-                except TimeoutError:
-                    continue
-                except Exception as e:
-                    return "Resolution '" + self.name + "' encountered an error: " + str(e)
-
-            page.close()
-            browser.close()
-        return returnResults

Modules/EDGAR_&_EFD/Edgar.xml

@@ -1,119 +0,0 @@
-<Edgar>
-    <Edgar_ID>
-        <Attributes>
-            <Attribute default="000000000000" check="Numbers" primary="True">CIK</Attribute>
-        </Attributes>
-    </Edgar_ID>
-    <Form_Field>
-        <Attributes>
-            <Attribute default="Form Field" check="String" primary="True">Field Name</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Account Number</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Fiscal Year</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Fiscal Period</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Value</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Unit</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Taxonomy</Attribute>
-        </Attributes>
-    </Form_Field>
-    <Form13F>
-        <Attributes>
-            <Attribute default="Form Field" check="String" primary="True">Name Of Issuer</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Title Of Class</Attribute>
-            <Attribute default="000000000" check="CUSIP" primary="False">CUSIP</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Value</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Number Of Shares</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Ssh Prnamt Type</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Investment Discretion</Attribute>
-        </Attributes>
-    </Form13F>
-    <Form4>
-        <Attributes>
-            <Attribute default="Security Title" check="String" primary="True">Security Title</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Deemed Execution Date</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Equity Swap Involved</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Transaction Timeliness</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Transaction Shares</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Transaction Price Per Share</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Shares Owned Following Transaction</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Direct Or Indirect Ownership</Attribute>
-        </Attributes>
-    </Form4>
-    <Form3>
-        <Attributes>
-            <Attribute default="Security Title" check="String" primary="True">Security Title</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Shares Owned Following Transaction</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Direct Or Indirect Ownership</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Nature Of Ownership</Attribute>
-        </Attributes>
-    </Form3>
-    <FormD>
-        <Attributes>
-            <Attribute default="Company Name" check="String" primary="True">Company Name</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Industry Group Type</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Investment Fund Type</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Aggregate Net Asset Value Range</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Duration Of Offering</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Types Of Securities Offered</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Business Combination Transaction</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Minimum Investment Accepted</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Total Offering Amount</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Total Amount Sold</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Total Amount Remaining</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Has Non Accredited Investors</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Total Number Already Invested</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Sales Commissions</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Finders Fees</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Gross Proceeds Used</Attribute>
-        </Attributes>
-    </FormD>
-    <Exchange>
-        <Attributes>
-            <Attribute default="Exchange Name" check="String" primary="True">Exchange Name</Attribute>
-        </Attributes>
-    </Exchange>
-    <SIC>
-        <Attributes>
-            <Attribute default="0000" check="SIC/NAICS" primary="True">SIC</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Description</Attribute>
-        </Attributes>
-    </SIC>
-    <EIN>
-        <Attributes>
-            <Attribute default="00-0000000" check="EIN" primary="True">EIN</Attribute>
-        </Attributes>
-    </EIN>
-    <CUSIP>
-        <Attributes>
-            <Attribute default="000000000" check="CUSIP" primary="True">CUSIP</Attribute>
-        </Attributes>
-    </CUSIP>
-    <LEIID>
-        <Attributes>
-            <Attribute default="00000000000000000000" check="LEIID" primary="True">LEIID</Attribute>
-        </Attributes>
-    </LEIID>
-    <ISINID>
-        <Attributes>
-            <Attribute default="000000000000" check="ISINID" primary="True">ISINID</Attribute>
-        </Attributes>
-    </ISINID>
-    <FormNMFP2>
-        <Attributes>
-            <Attribute default="NMFP2 Form Field" check="String" primary="True">Field Name</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Friday 1</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Friday 2</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Friday 3</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Friday 4</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Friday 5</Attribute>
-        </Attributes>
-    </FormNMFP2>
-    <Collateral_Issuer>
-        <Attributes>
-            <Attribute default="Issuer Name" check="String" primary="True">Name</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Coupon or Yield</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Principal Amount</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Value of Collateral</Attribute>
-            <Attribute default="Unknown" check="String" primary="False">Ctgry Investments Rprsnts Collateral</Attribute>
-        </Attributes>
-    </Collateral_Issuer>
-</Edgar>

Modules/EDGAR_&_EFD/FramesLookUp.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-
-class FramesLookUp:
-    # A string that is treated as the name of this resolution.
-    name = "Frames Look Up"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Frame Forms"
-
-    originTypes = {'Form Field'}
-
-    resultTypes = {'Edgar Company, Edgar ID, Country, Currency, Phrase'}
-
-    parameters = {
-        'Quarter': {'description': 'Please Ensure that the selected Taxonomy matches the Form Field you typed',
-                    'type': 'SingleChoice',
-                    'value': {'January, February, and March (Q1)', 'April, May, and June (Q2)', 'July, August, and '
-                                                                                                'September (Q3)',
-                              'October, November, and December (Q4)'}},
-        'Year': {'description': 'Please enter the year to match.',
-                 'type': 'String',
-                 'default': '2021'},
-        'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                       'Returns the 5 most recent by default.',
-                        'type': 'String',
-                        'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-
-        year = parameters['Year']
-        quarterChoice = parameters['Quarter']
-        quarter = quarterChoice[quarterChoice.find("(") + 1:quarterChoice.find(")")]
-        returnResults = []
-
-        for entity in entityJsonList:
-            uid = entity['uid']
-            unit = entity['Unit']
-            taxonomy = entity['Taxonomy']
-            form_field = entity['Field Name'].split(' ')[1]
-            search_url = f'https://data.sec.gov/api/xbrl/frames/{taxonomy}/{form_field}/{unit}/CY{year}{quarter}I.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-            data = r.json()
-            # print(data['data'])
-            if linkNumbers > len(data['data']):
-                linkNumbers = len(data['data'])
-
-            for i in range(linkNumbers):
-                # print(data['data'][i])
-                index_of_child = (len(returnResults))
-                returnResults.append([{'Company Name': data['data'][i]['entityName'],
-                                       'Entity Type': 'Company'},
-                                      {uid: {'Resolution': 'Edgar Company',
-                                             'Notes': ''}}])
-
-                returnResults.append([{'CIK': str(data['data'][i]['cik']).zfill(10),
-                                       'Entity Type': 'Edgar ID'},
-                                      {index_of_child: {'Resolution': '',
-                                                        'Notes': ''}}])
-
-                returnResults.append([{'Country Name': data['data'][i]['loc'],
-                                       'Entity Type': 'Country'},
-                                      {index_of_child: {'Resolution': '',
-                                                        'Notes': ''}}])
-                if unit == 'USD':
-                    returnResults.append([{'Amount': str(data['data'][i]['val']),
-                                           'Currency Type': 'USD',
-                                           'Entity Type': 'Currency'},
-                                          {index_of_child: {'Resolution': 'Form Filed Value',
-                                                            'Notes': ''}}])
-                elif unit == 'shares':
-                    returnResults.append([{'Phrase': 'Number of Shares: ' + str(data['data'][i]['val']),
-                                           'Entity Type': 'Phrase'},
-                                          {index_of_child: {'Resolution': 'Form Filed Value',
-                                                            'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/Get10KForms.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-
-class Get10KForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 10-K Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 10-K Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-            # print(data)
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '10-K' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 10-K: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Taxonomy': form,
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '10-K Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '10-K' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 10-K: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Taxonomy': form,
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '10-K Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get10QForms.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-
-class Get10QForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 10-Q Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 10-Q Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '10-Q' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 10-Q: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '10-Q Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '10-Q' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 10-Q: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '10-Q Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get13FForms.py

@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-
-class Get13FForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 13F Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 13F Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form13F'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns 5 more recent by default',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import xmltodict
-        import json
-        from bs4 import BeautifulSoup
-        from ast import literal_eval
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        name = ''
-        date = ''
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            archives_set = set()
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
-                         f'={linkNumbers}&type=13F-HR'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            soup = BeautifulSoup(r.text, "lxml")
-
-            for link in soup.find_all('a'):
-                # extract link url from the anchor
-                anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                if '/Archives/edgar/data/' in anchor:
-                    anchor = 'https://www.sec.gov' + anchor
-                    archives_set.add(anchor)
-
-            for archive in archives_set:
-                time.sleep(1)
-                r = requests.get(archive, headers=headers)
-                soup = BeautifulSoup(r.text, "lxml")
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-
-                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                    if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
-                            and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        r = requests.get(anchor, headers=headers)
-                        data = literal_eval(json.dumps(xmltodict.parse(r.text)))
-
-                        date = data['edgarSubmission']['headerData']['filerInfo']['periodOfReport']
-                        name = data['edgarSubmission']['formData']['coverPage']['filingManager']['name']
-
-                    elif '/Archives/edgar/data/' in anchor and 'infotable.xml' in anchor \
-                            and 'xslFormDX01' not in anchor and 'xslForm13F_X01' not in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        r = requests.get(anchor, headers=headers)
-                        data = literal_eval(json.dumps(xmltodict.parse(r.text)))
-
-                        for d in data['informationTable']['infoTable']:
-                            returnResults.append([{'Name Of Issuer': '13F-HR: ' + name + ' ' + d['nameOfIssuer'] + ' '
-                                                                     + date,
-                                                   'Title Of Class': d['titleOfClass'],
-                                                   'CUSIP': d['cusip'],
-                                                   'Value': d['value'],
-                                                   'Number Of Shares': d['shrsOrPrnAmt']['sshPrnamt'],
-                                                   'Ssh Prnamt Type': d['shrsOrPrnAmt']['sshPrnamtType'],
-                                                   'Investment Discretion': d['investmentDiscretion'],
-                                                   'Notes': '',
-
-                                                   'Entity Type': 'Form13F'},
-                                                  {uid: {'Resolution': 'Form13F',
-                                                         'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/Get20FForms.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python3
-
-class Get20FForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 20-F Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 20-F Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            # print(cik)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-            data = r.json()
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '20-F' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 20-F: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Taxonomy': form,
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '20-F Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '20-F' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 20-F: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Taxonomy': form,
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '20-F Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get3Forms.py

@@ -1,108 +0,0 @@
-#!/usr/bin/env python3
-
-class Get3Forms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 3 Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 3 Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Person, Form3'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import xmltodict
-        import json
-        from bs4 import BeautifulSoup
-        from ast import literal_eval
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            archives_set = set()
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
-                         f'={linkNumbers}&type=3'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            soup = BeautifulSoup(r.text, "lxml")
-
-            for link in soup.find_all('a'):
-                # extract link url from the anchor
-                anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                if '/Archives/edgar/data/' in anchor:
-                    anchor = 'https://www.sec.gov' + anchor
-                    archives_set.add(anchor)
-
-            for archive in archives_set:
-                time.sleep(1)
-                r = requests.get(archive, headers=headers)
-                soup = BeautifulSoup(r.text, "lxml")
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                    if '/Archives/edgar/data/' in anchor and 'ownership.xml' in anchor and 'xslF345X02' not in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        r = requests.get(anchor, headers=headers)
-                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
-                        data = literal_eval(data)
-                        # print(data)
-
-                        name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
-                        remarks = \
-                            data['ownershipDocument']['reportingOwner']['reportingOwnerRelationship']['officerTitle']
-                        value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeHolding']
-                        index_of_child = len(returnResults)
-                        returnResults.append([{'Full Name': name,
-                                               'Notes': remarks,
-                                               'Entity Type': 'Person'},
-                                              {uid: {'Resolution': 'Reporting Owner',
-                                                     'Notes': ''}}])
-
-                        if value['ownershipNature']['directOrIndirectOwnership']['value'] == 'I':
-                            nature = 'Indirect'
-                        else:
-                            nature = 'Direct'
-                        returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' ' +
-                                                                 data['ownershipDocument']['ownerSignature'][
-                                                                     'signatureDate'],
-                                               'Shares Owned Following Transaction':
-                                                   value['postTransactionAmounts']['sharesOwnedFollowingTransaction'][
-                                                       'value'],
-                                               'Direct Or Indirect Ownership': nature,
-                                               'Nature Of Ownership':
-                                                   value['ownershipNature']['natureOfOwnership']['value'],
-                                               'Notes': '',
-                                               'Entity Type': 'Form3'},
-                                              {index_of_child: {'Resolution': 'Form 3',
-                                                                'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get40FForms.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-
-class Get40FForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 40-F Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 40-F Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            # print(r.content)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '40-F' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 40-F: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Taxonomy': form,
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '40-F Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '40-F' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 40-F: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Taxonomy': form,
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '40-F Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get4Forms.py

@@ -1,149 +0,0 @@
-#!/usr/bin/env python3
-
-class Get4Forms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 4 Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes D Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Person, Form4'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import xmltodict
-        import json
-        from bs4 import BeautifulSoup
-        from ast import literal_eval
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            archives_set = set()
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
-                         f'={linkNumbers}&type=4'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            soup = BeautifulSoup(r.text, "lxml")
-
-            for link in soup.find_all('a'):
-                # extract link url from the anchor
-                anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                if '/Archives/edgar/data/' in anchor:
-                    anchor = 'https://www.sec.gov' + anchor
-                    archives_set.add(anchor)
-
-            for archive in archives_set:
-                time.sleep(1)
-                r = requests.get(archive, headers=headers)
-                soup = BeautifulSoup(r.text, "lxml")
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                    if '/Archives/edgar/data/' in anchor and '.xml' in anchor and 'xslF345X03' not in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        r = requests.get(anchor, headers=headers)
-                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
-                        data = literal_eval(data)
-                        # print(data)
-
-                        index_of_child = len(returnResults)
-                        try:
-                            remarks = data['ownershipDocument']['remarks']
-                        except KeyError:
-                            remarks = ''
-                        name = data['ownershipDocument']['reportingOwner']['reportingOwnerId']['rptOwnerName']
-                        returnResults.append([{'Full Name': name,
-                                               'Notes': remarks,
-                                               'Entity Type': 'Person'},
-                                              {uid: {'Resolution': 'Reporting Owner',
-                                                     'Notes': ''}}])
-
-                        if type(data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']) == dict:
-                            value = data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']
-                            try:
-                                footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
-                            except KeyError:
-                                footnote = value['transactionAmounts']['transactionShares']['value']
-                            try:
-                                footnotePerShare = \
-                                    value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
-                            except KeyError:
-                                footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
-                            returnResults.append([{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
-                                                                     + value['transactionCoding'][
-                                                                         'transactionCode'] + ' ' +
-                                                                     data['ownershipDocument']['ownerSignature'][
-                                                                         'signatureDate'],
-                                                   'Deemed Execution Date': str(value['deemedExecutionDate']),
-                                                   'Equity Swap Involved': value['transactionCoding'][
-                                                       'equitySwapInvolved'],
-                                                   'Transaction Timeliness': str(value['transactionTimeliness']),
-                                                   'Transaction Shares': footnote,
-                                                   'Transaction Price Per Share': footnotePerShare,
-                                                   'Shares Owned Following Transaction':
-                                                       value['postTransactionAmounts'][
-                                                           'sharesOwnedFollowingTransaction'],
-                                                   'Notes': (': '.join(
-                                                       map(str, data['ownershipDocument']['footnotes']['footnote']))),
-                                                   'Entity Type': 'Form4'},
-                                                  {index_of_child: {'Resolution': 'Form 4',
-                                                                    'Notes': ''}}])
-                        else:
-
-                            for value in data['ownershipDocument']['nonDerivativeTable']['nonDerivativeTransaction']:
-                                try:
-                                    footnote = value['transactionAmounts']['transactionShares']['footnoteId']['@id']
-                                except KeyError:
-                                    footnote = value['transactionAmounts']['transactionShares']['value']
-                                try:
-                                    footnotePerShare = \
-                                        value['transactionAmounts']['transactionPricePerShare']['footnoteId']['@id']
-                                except KeyError:
-                                    footnotePerShare = value['transactionAmounts']['transactionPricePerShare']['value']
-                                returnResults.append(
-                                    [{'Security Title': name + ': ' + value['securityTitle']['value'] + ' '
-                                                        + value['transactionCoding']['transactionCode'] + ' ' +
-                                                        data['ownershipDocument']['ownerSignature']['signatureDate'],
-                                      'Deemed Execution Date': str(value['deemedExecutionDate']),
-                                      'Equity Swap Involved': value['transactionCoding']['equitySwapInvolved'],
-                                      'Transaction Timeliness': str(value['transactionTimeliness']),
-                                      'Transaction Shares': footnote,
-                                      'Transaction Price Per Share': footnotePerShare,
-                                      'Shares Owned Following Transaction':
-                                          value['postTransactionAmounts']['sharesOwnedFollowingTransaction']['value'],
-                                      'Notes': (
-                                          ': '.join(map(str, data['ownershipDocument']['footnotes']['footnote']))),
-                                      'Entity Type': 'Form4'},
-                                     {index_of_child: {'Resolution': 'Form 4',
-                                                       'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/Get6KForms.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-class Get6KForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 6-K Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 6-K Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            # print(cik)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            # print(r.content)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-            # print(data)
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '6-K' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 6-K: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Taxonomy': form,
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '6-K Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '6-K' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 6-K: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Taxonomy': form,
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '6-K Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/Get8KForms.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-class Get8KForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent 8-K Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes 8-K Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Form Field'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            # print(cik)
-            search_url = f'https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            # print(r.content)
-            if r.status_code != 200:
-                return []
-
-            data = r.json()
-            # print(data)
-
-            forms = list(data['facts'].keys())
-
-            for form in forms:
-                keys = list(data['facts'][form].keys())
-                for i in keys:
-                    if 'Deprecated' not in data['facts'][form][i]['label']:
-                        if list(data['facts'][form][i]['units'].keys())[0] == 'USD':
-                            if linkNumbers > len(data['facts'][form][i]['units']['USD']):
-                                linkNumbers = int(len(data['facts'][form][i]['units']['USD']))
-                            for j in range(linkNumbers):
-                                if '8-K' in data['facts'][form][i]['units']['USD'][j]['form']:
-                                    value = data['facts'][form][i]['units']['USD'][::-1][j]
-                                    returnResults.append([{'Field Name': cik + ' 8-K: ' + i + ' ' + value['filed'],
-                                                           'Account Number': value['accn'],
-                                                           'Fiscal Year': value['fy'],
-                                                           'Fiscal Period': value['fp'],
-                                                           'Value': value['val'],
-                                                           'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                           'Taxonomy': form,
-                                                           'Notes': data['facts'][form][i]['label'],
-
-                                                           'Entity Type': 'Form Field'},
-                                                          {uid: {'Resolution': '8-K Field',
-                                                                 'Notes': ''}}])
-
-                    if list(data['facts'][form][i]['units'].keys())[0] == 'shares':
-                        if linkNumbers > len(data['facts'][form][i]['units']['shares']):
-                            linkNumbers = int(len(data['facts'][form][i]['units']['shares']))
-                        for j in range(linkNumbers):
-                            if '8-K' in data['facts'][form][i]['units']['shares'][j]['form']:
-                                value = data['facts'][form][i]['units']['shares'][::-1][j]
-                                returnResults.append([{'Field Name': cik + ' 8-K: ' + i + ' ' + value['filed'],
-                                                       'Account Number': value['accn'],
-                                                       'Fiscal Year': value['fy'],
-                                                       'Fiscal Period': value['fp'],
-                                                       'Value': value['val'],
-                                                       'Unit': list(data['facts'][form][i]['units'].keys())[0],
-                                                       'Taxonomy': form,
-                                                       'Notes': data['facts'][form][i]['label'],
-
-                                                       'Entity Type': 'Form Field'},
-                                                      {uid: {'Resolution': '8-K Field',
-                                                             'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/GetDForms.py

@@ -1,149 +0,0 @@
-#!/usr/bin/env python3
-
-class GetDForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent D Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes D Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'FormD, Person, Address, Phrase'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-        import xmltodict
-        import json
-        from bs4 import BeautifulSoup
-        from ast import literal_eval
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            archives_set = set()
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
-                         f'={linkNumbers}&type=D'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            soup = BeautifulSoup(r.text, "lxml")
-
-            for link in soup.find_all('a'):
-                # extract link url from the anchor
-                anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                if '/Archives/edgar/data/' in anchor:
-                    anchor = 'https://www.sec.gov' + anchor
-                    archives_set.add(anchor)
-
-            for archive in archives_set:
-                time.sleep(1)
-                r = requests.get(archive, headers=headers)
-                soup = BeautifulSoup(r.text, "lxml")
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                    if '/Archives/edgar/data/' in anchor and 'primary_doc.xml' in anchor \
-                            and 'xslFormDX01' not in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        r = requests.get(anchor, headers=headers)
-                        data = (json.dumps(xmltodict.parse(r.text))).replace('null', 'None')
-                        data = literal_eval(data)
-                        # print(data)
-
-                        value = data['edgarSubmission']['offeringData']
-                        index_of_child = len(returnResults)
-                        returnResults.append([{'Company Name': 'D: ' + data['edgarSubmission']['primaryIssuer']
-                        ['entityName'] + ' ' + value['signatureBlock']['signature']['signatureDate'],
-                                               'Industry Group Type': value['industryGroup']['industryGroupType'],
-                                               'Investment Fund Type': value['industryGroup']['investmentFundInfo']
-                                               ['investmentFundType'],
-                                               'Aggregate Net Asset Value Range': value['issuerSize']
-                                               ['aggregateNetAssetValueRange'],
-                                               'Duration Of Offering': 'More Than one Year: ' +
-                                                                       value['durationOfOffering']['moreThanOneYear'],
-                                               'Types Of Securities Offered': 'Pooled Investment Fund Type: ' +
-                                                                              value['typesOfSecuritiesOffered'][
-                                                                                  'isPooledInvestmentFundType'],
-                                               'Business Combination Transaction': 'Business Combination Transaction: '
-                                                                                   + value[
-                                                                                       'businessCombinationTransaction'][
-                                                                                       'isBusinessCombinationTransaction'],
-                                               'Minimum Investment Accepted': value['minimumInvestmentAccepted'],
-                                               'Total Offering Amount': value['offeringSalesAmounts']
-                                               ['totalOfferingAmount'],
-                                               'Total Amount Sold': value['offeringSalesAmounts']['totalAmountSold'],
-                                               'Total Amount Remaining': value['offeringSalesAmounts'][
-                                                   'totalRemaining'],
-                                               'Has Non Accredited Investors': 'Non Accredited Investors'
-                                                                               + value['investors'][
-                                                                                   'hasNonAccreditedInvestors'],
-                                               'Total Number Already Invested': value['investors']
-                                               ['totalNumberAlreadyInvested'],
-                                               'Sales Commissions': value['salesCommissionsFindersFees']
-                                               ['salesCommissions']['dollarAmount'],
-                                               'Finders Fees': value['salesCommissionsFindersFees']['findersFees']
-                                               ['dollarAmount'],
-                                               'Gross Proceeds Used': value['useOfProceeds']['grossProceedsUsed']
-                                               ['dollarAmount'],
-                                               'Notes': '',
-
-                                               'Entity Type': 'FormD'},
-                                              {uid: {'Resolution': 'D Form',
-                                                     'Notes': ''}}])
-
-                        people = data['edgarSubmission']['relatedPersonsList']['relatedPersonInfo']
-                        for person in people:
-                            child_of_child = len(returnResults)
-                            returnResults.append([{'Full Name': person['relatedPersonName']['firstName'] + ' ' +
-                                                                person['relatedPersonName']['lastName'],
-                                                   'Entity Type': 'Person'},
-                                                  {index_of_child: {'Resolution': 'Officer',
-                                                                    'Notes': ''}}])
-
-                            returnResults.append(
-                                [{'Street Address': person['relatedPersonAddress']['street1'],
-                                  'Locality': person['relatedPersonAddress']['city'],
-                                  'Postal Code': person['relatedPersonAddress']['zipCode'],
-                                  'Country': person['relatedPersonAddress']['stateOrCountryDescription'],
-                                  'Entity Type': 'Address'},
-                                 {child_of_child: {'Resolution': 'Location', 'Notes': ''}}])
-
-                            returnResults.append(
-                                [{'Phrase': person['relatedPersonRelationshipList']['relationship'],
-                                  'Entity Type': 'Phrase'},
-                                 {child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
-
-                            if person['relationshipClarification'] is not None:
-                                returnResults.append(
-                                    [{'Phrase': person['relationshipClarification'],
-                                      'Entity Type': 'Phrase'},
-                                     {child_of_child: {'Resolution': 'Relationship', 'Notes': ''}}])
-
-        return returnResults

Modules/EDGAR_&_EFD/GetN8FForms.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-class GetN8FForms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent N-8F Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes N-8F Forms Websites"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Website'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import time
-
-        from bs4 import BeautifulSoup
-
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
-        }
-
-        try:
-            linkNumbers = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer provided in 'Max Results' parameter"
-        if linkNumbers <= 0:
-            return []
-        returnResults = []
-        for entity in entityJsonList:
-            archives_set = set()
-            uid = entity['uid']
-            cik = entity['CIK']
-            if cik.lower().startswith('cik'):
-                cik = cik.split('cik')[1]
-            if len(cik) != 10:
-                cik = cik.zfill(10)
-            search_url = f'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&owner=include&count' \
-                         f'={linkNumbers}&type=N-8F'
-            time.sleep(1)
-            r = requests.get(search_url, headers=headers)
-            if r.status_code != 200:
-                return []
-
-            soup = BeautifulSoup(r.text, "lxml")
-
-            for link in soup.find_all('a'):
-                # extract link url from the anchor
-                anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                if '/Archives/edgar/data/' in anchor:
-                    anchor = 'https://www.sec.gov' + anchor
-                    archives_set.add(anchor)
-
-            for archive in archives_set:
-                time.sleep(1)
-                r = requests.get(archive, headers=headers)
-                soup = BeautifulSoup(r.text, "lxml")
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-                    anchor = link.attrs['href'] if 'href' in link.attrs else ''
-                    if '/Archives/edgar/data/' in anchor and '.htm' in anchor:
-                        time.sleep(1)
-                        anchor = 'https://www.sec.gov' + anchor
-                        returnResults.append([{'URL': anchor,
-                                               'Entity Type': 'Website'},
-                                              {uid: {'Resolution': 'N-8F Form',
-                                                     'Notes': ''}}])
-        return returnResults

Modules/EDGAR_&_EFD/GetNMFP2Forms.py

@@ -1,275 +0,0 @@
-#!/usr/bin/env python3
-
-class GetNMFP2Forms:
-    # A string that is treated as the name of this resolution.
-    name = "Get Recent N-MFP2 Forms"
-
-    category = "EDGAR Info"
-
-    # A string that describes this resolution.
-    description = "Returns Nodes N-MFP2 Forms"
-
-    originTypes = {'Edgar ID'}
-
-    resultTypes = {'Collateral Issuer, Company, Phrase, FormNMFP2, CUSIP, LEIID, ISINID'}
-
-    parameters = {'Max Results': {'description': 'Please enter the maximum number of results to return.\n'
-                                                 'Returns the 5 most recent by default.',
-                                  'type': 'String',
-                                  'default': '5'}}
-
-    def resolution(self, entityJsonList, parameters):
-        import requests
-        import xmltodict
-        import json
-        from playwright.sync_api import sync_playwright, TimeoutError, Error
-        from bs4 import BeautifulSoup
-        from ast import literal_eval
-
-        headers = {
-            'User-Agent': 'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
-        }
-
-        try:
-            maxResults = int(parameters['Max Results'])
-        except ValueError:
-            return "Invalid integer value provided for 'Max Results' parameter."
-        if maxResults <= 0:
-            return []
-
-        returnResults = []
-        liquidAssets = ['totalValueDailyLiquidAssets', 'totalValueWeeklyLiquidAssets', 'percentageDailyLiquidAssets',
-                        'percentageWeeklyLiquidAssets', 'netAssetValue']
-        seriesLevelInfoKeys = ['feederFundFlag', 'masterFundFlag', 'seriesFundInsuCmpnySepAccntFlag',
-                               'fundExemptRetailFlag', 'averagePortfolioMaturity',
-                               'averageLifeMaturity', 'cash',
-                               'totalValuePortfolioSecurities', 'amortizedCostPortfolioSecurities',
-                               'totalValueOtherAssets', 'totalValueLiabilities', 'netAssetOfSeries',
-                               'numberOfSharesOutstanding', 'stablePricePerShare', 'sevenDayGrossYield']
-        classLevelInfoKeys = ['minInitialInvestment', 'netAssetsOfClass', 'numberOfSharesOutstanding',
-                              'sevenDayNetYield', 'personPayForFundFlag']
-        securitiesInfoKeys = ['titleOfIssuer', 'investmentCategory', 'securityEligibilityFlag',
-                              'investmentMaturityDateWAM', 'investmentMaturityDateWAL',
-                              'finalLegalInvestmentMaturityDate', 'securityDemandFeatureFlag', 'securityGuaranteeFlag',
-                              'securityEnhancementsFlag', 'yieldOfTheSecurityAsOfReportingDate',
-                              'includingValueOfAnySponsorSupport', 'excludingValueOfAnySponsorSupport',
-                              'percentageOfMoneyMarketFundNetAssets', 'securityCategorizedAtLevel3Flag',
-                              'dailyLiquidAssetSecurityFlag', 'weeklyLiquidAssetSecurityFlag', 'illiquidSecurityFlag']
-
-        with sync_playwright() as p:
-            browser = p.firefox.launch()
-            context = browser.new_context(
-                viewport={'width': 1920, 'height': 1080},
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0'
-            )
-            page = context.new_page()
-            for entity in entityJsonList:
-                archives_set = set()
-                uid = entity['uid']
-                cik = entity['CIK']
-                if cik.lower().startswith('cik'):
-                    cik = cik.split('cik')[1]
-                if len(cik) != 10:
-                    cik = cik.zfill(10)
-                search_url = f'https://www.sec.gov/edgar/search/#/category=custom&entityName={cik}&forms=N-MFP2'
-                page.wait_for_timeout(1000)
-                pageResolved = False
-                for _ in range(3):
-                    try:
-                        page.goto(search_url, wait_until="networkidle", timeout=10000)
-                        pageResolved = True
-                        break
-                    except TimeoutError:
-                        pass
-                    except Error:
-                        break
-                if not pageResolved:
-                    continue
-                page.wait_for_timeout(1000)
-
-                soup = BeautifulSoup(page.content(), "lxml")
-
-                for link in soup.find_all('a'):
-                    # extract link url from the anchor
-                    anchor = link.attrs['data-adsh'] if 'data-adsh' in link.attrs else ''
-                    if anchor != '':
-                        anchor = anchor.replace('-', '')
-                        anchor = f'https://www.sec.gov/Archives/edgar/data/{cik}/{anchor}/primary_doc.xml'
-                        archives_set.add(anchor)
-
-                for link in range(maxResults):
-                    r = requests.get(list(archives_set)[link], headers=headers)
-                    data = literal_eval(json.dumps(xmltodict.parse(r.text)).replace('null', 'None'))
-                    fieldPath = data['edgarSubmission']['formData']['seriesLevelInfo']
-                    seriesId = data['edgarSubmission']['formData']['generalInfo'][
-                        'seriesId']
-                    date = data['edgarSubmission']['formData']['generalInfo'][
-                        'reportDate']
-
-                    returnResults.append(
-                        [{'Company Name': fieldPath['adviser']['adviserName'],
-                          'Entity Type': 'Company'},
-                         {uid: {'Resolution': 'Adviser', 'Notes': ''}}])
-                    returnResults.append(
-                        [{'Company Name': fieldPath['indpPubAccountant']['name'],
-                          'Entity Type': 'Company'},
-                         {uid: {'Resolution': 'Independent Pub Accountant', 'Notes': ''}}])
-                    returnResults.append(
-                        [{'Company Name': fieldPath['administrator']['administratorName'],
-                          'Entity Type': 'Company'},
-                         {uid: {'Resolution': 'Administrator', 'Notes': ''}}])
-                    returnResults.append(
-                        [{'Company Name': fieldPath['transferAgent']['name'],
-                          'Entity Type': 'Company'},
-                         {uid: {'Resolution': 'Transfer Agent', 'Notes': ''}}])
-
-                    for value in seriesLevelInfoKeys:
-                        returnResults.append(
-                            [{'Phrase': f'N-MFP2:({value}) '
-                                        + f'ID: {seriesId} Date: {date}',
-                              'Notes': fieldPath[value],
-                              'Entity Type': 'Phrase'},
-                             {uid: {'Resolution': value, 'Notes': ''}}])
-
-                    for value in fieldPath['moneyMarketFundCategory']:
-                        returnResults.append(
-                            [{'Phrase': value,
-                              'Entity Type': 'Phrase'},
-                             {uid: {'Resolution': value, 'Notes': ''}}])
-
-                    for field in liquidAssets:
-                        if 'Daily' in field:
-                            returnResults.append([{'Field Name': cik + f' N-MFP2:({field})' + ' '
-                                                                 + f'ID: {seriesId} Date: {date}',
-                                                   'Friday 1': fieldPath[field][
-                                                       'ns3:fridayDay1'],
-                                                   'Friday 2': fieldPath[field][
-                                                       'ns3:fridayDay2'],
-                                                   'Friday 3': fieldPath[field][
-                                                       'ns3:fridayDay3'],
-                                                   'Friday 4': fieldPath[field][
-                                                       'ns3:fridayDay4'],
-                                                   'Friday 5': 'NO Value in Daily Measure',
-                                                   'Entity Type': 'FormNMFP2'},
-                                                  {uid: {'Resolution': field,
-                                                         'Notes': ''}}])
-                        else:
-                            returnResults.append([{'Field Name': cik + f' N-MFP2:({field})' + ' '
-                                                                 + f'ID: {seriesId} Date: {date}',
-                                                   'Friday 1': fieldPath[field][
-                                                       'ns3:fridayWeek1'],
-                                                   'Friday 2': fieldPath[field][
-                                                       'ns3:fridayWeek2'],
-                                                   'Friday 3': fieldPath[field][
-                                                       'ns3:fridayWeek3'],
-                                                   'Friday 4': fieldPath[field][
-                                                       'ns3:fridayWeek4'],
-                                                   'Friday 5': fieldPath[field][
-                                                       'ns3:fridayWeek5'],
-                                                   'Entity Type': 'FormNMFP2'},
-                                                  {uid: {'Resolution': field,
-                                                         'Notes': ''}}])
-
-                    classLevelInfo = data['edgarSubmission']['formData']['classLevelInfo']
-                    for classInfo in classLevelInfo:
-                        index_of_child = len(returnResults)
-                        returnResults.append(
-                            [{'Phrase': classInfo['classesId'],
-                              'Entity Type': 'Phrase'},
-                             {uid: {'Resolution': 'Classes Id', 'Notes': ''}}])
-                        returnResults.append([{'Field Name': cik + f' N-MFP2:(Net Asset Per Share)' + ' '
-                                                             + f'ID: {seriesId} Date: {date}',
-                                               'Friday 1': classInfo['netAssetPerShare'][
-                                                   'ns3:fridayWeek1'],
-                                               'Friday 2': classInfo['netAssetPerShare'][
-                                                   'ns3:fridayWeek2'],
-                                               'Friday 3': classInfo['netAssetPerShare'][
-                                                   'ns3:fridayWeek3'],
-                                               'Friday 4': classInfo['netAssetPerShare'][
-                                                   'ns3:fridayWeek4'],
-                                               'Friday 5': classInfo['netAssetPerShare'][
-                                                   'ns3:fridayWeek5'],
-                                               'Entity Type': 'FormNMFP2'},
-                                              {index_of_child: {'Resolution': 'Net Asset Per Share',
-                                                                'Notes': ''}}])
-                        for weekCount in range(1, 6):
-                            returnResults.append(
-                                [{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossSubscriptions'],
-                                  'Entity Type': 'Phrase'},
-                                 {index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Subscriptions',
-                                                   'Notes': ''}}])
-                            returnResults.append(
-                                [{'Phrase': classInfo[f'fridayWeek{weekCount}']['weeklyGrossRedemptions'],
-                                  'Entity Type': 'Phrase'},
-                                 {index_of_child: {'Resolution': f'Friday Week {weekCount} Weekly Gross Redemptions',
-                                                   'Notes': ''}}])
-
-                        for value in classLevelInfoKeys:
-                            returnResults.append(
-                                [{'Phrase': classInfo[value],
-                                  'Entity Type': 'Phrase'},
-                                 {index_of_child: {'Resolution': value,
-                                                   'Notes': ''}}])
-
-                    scheduleOfPortfolioSecurities = data['edgarSubmission']['formData'][
-                        'scheduleOfPortfolioSecuritiesInfo']
-                    instance = 0
-                    for securitiesInfo in scheduleOfPortfolioSecurities:
-                        index_of_child = len(returnResults)
-                        returnResults.append(
-                            [{'Company Name': securitiesInfo.get('nameOfIssuer') + ' ' + str(instance),
-                              'Entity Type': 'Company'},
-                             {uid: {'Resolution': 'Issuer', 'Notes': ''}}])
-                        instance += 1
-                        returnResults.append(
-                            [{'CUSIP': securitiesInfo.get('CUSIPMember'),
-                              'Entity Type': 'CUSIP'},
-                             {index_of_child: {'Resolution': 'CUSIP', 'Notes': ''}}])
-                        returnResults.append(
-                            [{'LEIID': securitiesInfo.get('LEIID'),
-                              'Entity Type': 'LEIID'},
-                             {index_of_child: {'Resolution': 'LEIID', 'Notes': ''}}])
-                        returnResults.append(
-                            [{'ISINID': securitiesInfo.get('ISINId'),
-                              'Entity Type': 'ISINID'},
-                             {index_of_child: {'Resolution': 'ISINID', 'Notes': ''}}])
-
-                        for value in securitiesInfoKeys:
-                            returnResults.append(
-                                [{'Phrase': securitiesInfo[value],
-                                  'Entity Type': 'Phrase'},
-                                 {index_of_child: {'Resolution': value,
-                                                   'Notes': ''}}])
-
-                        for value in securitiesInfo['NRSRO']:
-                            child_of_child = len(returnResults)
-                            returnResults.append(
-                                [{'Company Name': value.get('nameOfNRSRO'),
-                                  'Entity Type': 'Company'},
-                                 {index_of_child: {'Resolution': 'NRSRO',
-                                                   'Notes': ''}}])
-                            returnResults.append(
-                                [{'Phrase': value.get('rating'),
-                                  'Entity Type': 'Phrase'},
-                                 {child_of_child: {'Resolution': 'Rating',
-                                                   'Notes': ''}}])
-
-                        try:
-                            collateralIssuer = securitiesInfo['collateralIssuers']
-                            for issuer in collateralIssuer:
-                                returnResults.append([{'Name': issuer['nameOfCollateralIssuer'],
-                                                       'Coupon or Yield': issuer['couponOrYield'],
-                                                       'Principal Amount': issuer['principalAmountToTheNearestCent'],
-                                                       'Value of Collateral': issuer[
-                                                           'valueOfCollateralToTheNearestCent'],
-                                                       'Ctgry Investments Rprsnts Collateral':
-                                                           issuer['ctgryInvestmentsRprsntsCollateral'],
-                                                       'Entity Type': 'Collateral Issuer'},
-                                                      {index_of_child: {'Resolution': 'Collateral Issuer',
-                                                                        'Notes': ''}}])
-
-                        except KeyError:
-                            continue
-            page.close()
-            browser.close()
-        return returnResults

Modules/EDGAR_&_EFD/requirements.txt

@@ -1,5 +0,0 @@
-requests
-bs4
-xmltodict
-playwright
-datetime

Modules/ExampleModule/ExampleModuleEntities.xml

@@ -1,10 +0,0 @@
-<ExampleEntities>
-  <Example>
-    <Attributes>
-      <Attribute default="Default value for ExampleLabel attribute" check="String" primary="True">ExampleLabel</Attribute>
-    </Attributes>
-    <Icon>
-      Default.svg
-    </Icon>
-  </Example>
-</ExampleEntities>

Modules/ExampleModule/ExampleResolution.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-
-class ExampleResolution:
-    # A string that is treated as the name of this resolution.
-    name = "Example Resolution"
-
-    category = "Example"
-
-    # A string that describes this resolution.
-    description = "Resolves Nothing in particular"
-
-    # A set of entities that this resolution can be ran on.
-    originTypes = {'Person'}
-
-    # A set of entities that could be the result of this resolution.
-    resultTypes = {'Person'}
-
-    # A dictionary of properties for this resolution. The key is the property name,
-    # the value is the property attributes. The type of input expected from the user is determined by the
-    # variable type of the 'value' parameter.
-    parameters = {'String Example': {'description': 'Example String Description',
-                                     'type': 'String',
-                                     'value': ''},
-
-                  'File Example': {'description': 'Example Choose File Description',
-                                   'type': 'File',
-                                   'value': ''},
-
-                  'Choose One Example': {'description': 'Example Choose One Description',
-                                         'type': 'SingleChoice',
-                                         'value': {'one', 'two', 'three'}
-                                         },
-
-                  'Choose Multiple Example': {'description': 'Example Choose Multiple Description',
-                                              'type': 'MultiChoice',
-                                              'value': {'one', 'two', 'three'}
-                                              }
-                  }
-
-    def resolution(self, entityJsonList, parameters):
-        """
-        eJsonList is a dictionary where the keys are the accepted origin
-        types, and the values are lists of json representations of
-        entities whose type matches the key.
-
-        parameters is a dictionary with the keys of the 'parameters' variable.
-        The value of each key is the user's input.
-
-        Example:
-        If the origin types of an entity are: {'Person', 'Alias'}
-
-        The input could be:
-        [Person1JSON, Person2JSON]
-
-        or:
-
-        [Person1JSON, Alias1JSON]
-
-        Returns a list of lists, where each inner list contains an entity produced as output, and a dict of dicts
-        where the keys of the outer dictionary are either UIDs of input nodes or indices of elements in the outer list.
-        The inner dict holds the 'Resolution' and 'Notes' characteristics of the link to create.
-
-        Example:
-
-        [[resultNodeJson1, {'inputNodeUID1': {'Resolution': 'LinkName1', 'Notes': 'LinkNotes1'},
-                            resultNodeIndex1: {'Resolution': 'LinkName2', 'Notes': ''}}],
-        ...]
-        """
-        return []