More work on ModuleManager.

Core/ModuleManager.py

@@ -29,37 +29,34 @@ class ModulesManager:
         self.baseAppStoragePath.mkdir(exist_ok=True, parents=True)
         self.modulesBaseDirectoryPath = self.baseAppStoragePath / "User Module Packs Storage"
         self.modulesBaseDirectoryPath.mkdir(exist_ok=True)
+        self.browsersBaseDirectoryPath = self.baseAppStoragePath / "Browsers"
+        self.browsersBaseDirectoryPath.mkdir(exist_ok=True)
         self.modulesRequirementsPath = self.modulesBaseDirectoryPath / "requirements.txt"
         self.modulesRequirementsPath.touch(mode=0o700, exist_ok=True)
         self.modulesRequirementsTempPath = self.modulesBaseDirectoryPath / "requirements.txt.tmp"
-        self.modulesRequirementsTrackingPath = self.modulesBaseDirectoryPath / "Module Requirements Tracking.yml"
-        self.modulesRequirementsTrackingPath.touch(mode=0o700, exist_ok=True)
         self.modulesPythonPath = self.modulesBaseDirectoryPath / 'bin' / 'python3'
         self.upgradeThread = None
         self.moduleReqsThread = None
+        self.uninstallThread = None
+        self.modulePacksListViewer = None
         self.upgradeLock = Lock()
-        self.sources = {}
+        self.sources = self.mainWindow.SETTINGS.value("Program/Sources/Sources List")
-        self.modulePacks = {}
+        self.modulePacks = self.mainWindow.SETTINGS.value("Program/Sources/Module Packs List")
         self.loadedModules = {}
 
-        self.load()
-
         self.venvThread = InitialiseVenvThread(self)
         self.venvThread.configureVenvOfMainThreadSignal.connect(self.configureVenv)
         self.venvThread.start()
 
     def afterUpgrade(self, upgradeStatus: bool):
         self.mainWindow.MESSAGEHANDLER.info(f"Upgrade status: {'Success' if upgradeStatus else 'Failed'}")
+        self.mainWindow.setStatus('Environment Updated.')
 
     def save(self) -> bool:
         self.mainWindow.SETTINGS.setValue("Program/Sources/Sources List", self.sources)
         self.mainWindow.SETTINGS.setValue("Program/Sources/Module Packs List", self.modulePacks)
         return True
 
-    def load(self):
-        self.sources = self.mainWindow.SETTINGS.value("Program/Sources/Sources List")
-        self.modulePacks = self.mainWindow.SETTINGS.value("Program/Sources/Module Packs List")
-
     def loadYamlFile(self, filePath: Path):
         with open(filePath, 'r') as yamlFile:
             try:
@@ -78,6 +75,7 @@ class ModulesManager:
         # prepend bin to PATH (this file is inside the bin directory)
         os.environ["PATH"] = os.pathsep.join([binDir] + os.environ.get("PATH", "").split(os.pathsep))
         os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
+        os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(self.browsersBaseDirectoryPath)
 
         # add the virtual environments libraries to the host python import mechanism
         prevLength = len(sys.path)
@@ -239,6 +237,7 @@ class ModulesManager:
         else:
             try:
                 shutil.copytree(Path(sourceURI), destinationPath, dirs_exist_ok=True)
+                self.mainWindow.MESSAGEHANDLER.info('Copied source files.')
             except FileNotFoundError:
                 if showMessages:
                     self.mainWindow.MESSAGEHANDLER.error(f'Could not sync source {sourceURI}: Source folder not found.',
@@ -268,7 +267,9 @@ class ModulesManager:
         return True
 
     def showModuleManager(self):
-        ModulePacksListViewer(self).exec()
+        self.modulePacksListViewer = ModulePacksListViewer(self)
+        self.modulePacksListViewer.exec()
+        self.modulePacksListViewer = None
 
 
     def installModulePackInit(self, sourceUUID: str, modulePackUUID: str) -> None:
@@ -288,83 +289,48 @@ class ModulesManager:
         self.moduleReqsThread.installFinishedSignal.connect(self.installModulePackFin)
         self.moduleReqsThread.start()
 
-    def installModulePackFin(self, success: bool, modulePackLabel: str):
+    def installModulePackFin(self, success: bool, modulePackLabel: str, sourceUID: str, modulePackUID: str):
         self.upgradeLock.release()
         if success:
             self.mainWindow.MESSAGEHANDLER.info(f'Module Pack successfully installed: {modulePackLabel}')
+            with contextlib.suppress(AttributeError):
+                self.modulePacksListViewer.setInstallCompletedForModulePackWidget()
+            modulePackDetails = self.modulePacks[sourceUID][modulePackUID]
+            for module in modulePackDetails['modules']:
+                modulePath = self.modulesBaseDirectoryPath / sourceUID / module
+                if not self.loadModule(modulePath):
+                    self.mainWindow.MESSAGEHANDLER.warning(
+                        f'Failed loading module: {modulePackLabel}.',
+                        exc_info=False,
+                    )
+            self.mainWindow.reloadModules(onlyUpdateDockbar=True)
         else:
             self.mainWindow.MESSAGEHANDLER.info(f'Module Pack installation failed: {modulePackLabel}')
 
-    def removeModulePack(self, modulePackDict: dict) -> bool:
+    def removeModulePack(self, modulePackDict: dict) -> None:
-        print('Rem', modulePackDict)
+        self.upgradeLock.acquire()
-        if not modulePackDict['Installed']:
-            return True
 
-    def installModule(self, uniqueModuleName: str, moduleFilePath: Path) -> bool:
+        self.uninstallThread = UninstallModuleThread(self, modulePackDict)
-        newModuleDirectoryPath = self.modulesDirectoryPath / uniqueModuleName
-        if newModuleDirectoryPath.exists():
-            self.mainWindow.MESSAGEHANDLER.error(f'Could not install Module {uniqueModuleName}: Module already exists.',
-                                                 popUp=True, exc_info=False)
-            return False
-        shutil.copytree(moduleFilePath, newModuleDirectoryPath, dirs_exist_ok=True)
 
-        moduleRequirements = newModuleDirectoryPath / "requirements.txt"
+        progress = QtWidgets.QProgressDialog(f'Uninstalling Module: {modulePackDict["label"]}...',
-        moduleAssetsPath = newModuleDirectoryPath / "assets"
+                                             '', 0, 3, self.mainWindow)
-        resolutionsPath = newModuleDirectoryPath / "Resolutions"
-        entitiesPath = newModuleDirectoryPath / "Entities"
-        moduleAssetsPath.mkdir(exist_ok=True)
-        resolutionsPath.mkdir(exist_ok=True)
-        entitiesPath.mkdir(exist_ok=True)
 
-        moduleDetailsPath = newModuleDirectoryPath / "module.yml"
+        # Remove Cancel button from progress bar (user should not be able to stop canvas from loading).
-        if not moduleDetailsPath.exists():
+        progress.setMinimumDuration(0)
-            moduleDetailsPath = newModuleDirectoryPath / "module.yaml"
+        progress.setCancelButton(None)
-            if not moduleDetailsPath.exists():
+        progress.setWindowModality(QtCore.Qt.WindowModality.WindowModal)
-                self.mainWindow.MESSAGEHANDLER.error(f'Could not install Module {uniqueModuleName}: '
+        self.uninstallThread.progressSignal.connect(progress.setValue)
-                                                     f'module.yml not found.')
+        self.uninstallThread.uninstallFinishedSignal.connect(self.uninstallModulePackFin)
-            shutil.rmtree(newModuleDirectoryPath)
+        self.uninstallThread.start()
-            return False
 
-        if not (moduleDetails := self.loadYamlFile(moduleDetailsPath)):
+    def uninstallModulePackFin(self, success: bool, modulePackLabel: str):
-            shutil.rmtree(newModuleDirectoryPath)
+        self.upgradeLock.release()
-            return False
+        if success:
-
+            self.mainWindow.MESSAGEHANDLER.info(f'Module Pack successfully uninstalled: {modulePackLabel}')
-        try:
+            with contextlib.suppress(AttributeError):
-            author = moduleDetails['Author']
+                self.modulePacksListViewer.setUninstallCompletedForModulePackWidget()
-            version = moduleDetails['Version']
+        else:
-            moduleName = moduleDetails['Module Name']
+            self.mainWindow.MESSAGEHANDLER.info(f'Module Pack uninstallation failed: {modulePackLabel}')
-            notes = moduleDetails['Notes']
-        except Exception as exc:
-            self.mainWindow.MESSAGEHANDLER.error(f'Could not install Module {uniqueModuleName}. '
-                                                 f'Exception while loading module details: {exc}')
-            shutil.rmtree(newModuleDirectoryPath)
-            return False
-
-        if moduleRequirements.exists():
-            cmdStr = f"'{self.modulesPythonPath}' -m pip install -r '{moduleRequirements}'"
-            subprocess.run(cmdStr, shell=True)
-
-        self.mainWindow.MESSAGEHANDLER.info(f'Loaded module {moduleName} version {version} by {author}.')
-        self.mainWindow.MESSAGEHANDLER.debug(f'Module {moduleName} notes: {notes}')
-        return True
-
-    def uninstallModule(self, uniqueModuleName: str) -> bool:
-        newModuleDirectoryPath = self.modulesDirectoryPath / uniqueModuleName
-        if not newModuleDirectoryPath.exists():
-            self.mainWindow.MESSAGEHANDLER.error(f'Could not uninstall Module {uniqueModuleName}: '
-                                                 f'Module does not exist.')
-            self.modules.pop(uniqueModuleName)
-            return False
-        try:
-            shutil.rmtree(newModuleDirectoryPath)
-        except Exception as exc:
-            self.mainWindow.MESSAGEHANDLER.error(f'Could not uninstall Module {uniqueModuleName}: '
-                                                 f'Error occurred: {exc}.')
-            return False
-
-        # No need to uninstall anything from venv - more likely to cause issues than fix anything.
-        self.modules.pop(uniqueModuleName)
-        return True
 
 
 class SourcesManager(QtWidgets.QDialog):
@@ -652,13 +618,18 @@ class ModulePacksListViewer(QtWidgets.QDialog):
         layout.addWidget(addModulePackButton, 5, 1, 1, 1)
         layout.addWidget(closeButton, 6, 0, 1, 2)
 
+        # Sort module packs alphabetically.
+        allPacksDict = {}
         for source in self.modulesManager.modulePacks:
             for module, moduleDetails in self.modulesManager.modulePacks[source].items():
                 newItemWidget = ModulePacksListItem(self, moduleDetails)
                 newItem = QtWidgets.QListWidgetItem()
                 newItem.setSizeHint(newItemWidget.sizeHint())
-                self.modulePackList.addItem(newItem)
+                allPacksDict[newItemWidget] = (moduleDetails['label'], newItem)
-                self.modulePackList.setItemWidget(newItem, newItemWidget)
+        allPacksDict = dict(sorted(allPacksDict.items(), key=lambda item: item[1][0]))
+        for itemWidget, value in allPacksDict.items():
+            self.modulePackList.addItem(value[1])
+            self.modulePackList.setItemWidget(value[1], itemWidget)
         self.setMinimumWidth(350)
         layout.setRowStretch(3, 10)
 
@@ -670,6 +641,8 @@ class ModulePacksListViewer(QtWidgets.QDialog):
             return True
 
         self.modulesManager.removeModulePack(widgetOfItem.modulePackDict)
+        self.moduleWidgetBeingInstalled = widgetOfItem
+        widgetOfItem.installedLabel.setText("Uninstalling...")
 
     def addModulePackPrompt(self) -> None:
         try:
@@ -683,7 +656,7 @@ class ModulePacksListViewer(QtWidgets.QDialog):
                 f'Module "{widgetOfItem.modulePackDict["label"]}" is already Installed.', popUp=True, exc_info=False)
         elif AddModulePackDialog(self, widgetOfItem.modulePackDict, False).exec():
             self.moduleWidgetBeingInstalled = widgetOfItem
-            widgetOfItem.installedLabel.setText("Installed: Pending")  # TODO
+            widgetOfItem.installedLabel.setText("Installed: Pending")
 
     def installModulePack(self, sourceUUID: str, modulePackUUID: str) -> None:
         packDetails = self.modulesManager.modulePacks[sourceUUID][modulePackUUID]
@@ -696,6 +669,12 @@ class ModulePacksListViewer(QtWidgets.QDialog):
             self.moduleWidgetBeingInstalled.installedLabel.setText("Installed: True")
             self.moduleWidgetBeingInstalled = None
 
+    def setUninstallCompletedForModulePackWidget(self):
+        if self.moduleWidgetBeingInstalled is not None:
+            self.moduleWidgetBeingInstalled.installedLabel.setText("Installed: False")
+            self.moduleWidgetBeingInstalled = None
+
+
 
 class ModulePacksListItem(QtWidgets.QWidget):
 
@@ -801,11 +780,6 @@ class InitialiseVenvThread(QtCore.QThread):
 
         self.configureVenvOfMainThreadSignal.emit(venvPath)
 
-        # Install / upgrade playwright & misc if not already installed, since they need special treatment.
-        cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m pip install --upgrade wheel setuptools playwright && " \
-                 f"'{self.modulesManager.modulesPythonPath}' -m playwright install"
-        subprocess.run(cmdStr, shell=True)
-
 
 class UpgradeVenvThread(QtCore.QThread):
     upgradeVenvThreadSignal = QtCore.Signal(bool)
@@ -824,9 +798,16 @@ class UpgradeVenvThread(QtCore.QThread):
             reqsFile = self.modulesManager.modulesRequirementsTempPath
         else:
             reqsFile = self.modulesManager.modulesRequirementsPath
-        cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m pip install --upgrade -r '{reqsFile}'"
         try:
+            cmdStr = f"'{self.modulesManager.modulesPythonPath}' --version"
             subprocess.check_output(cmdStr, shell=True)
+            cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m pip install --upgrade -r '{reqsFile}'"
+            subprocess.check_output(cmdStr, shell=True)
+            # Install / upgrade playwright & misc if not already installed, since they need special treatment.
+            cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m pip install --upgrade pip wheel setuptools playwright"
+            subprocess.run(cmdStr, shell=True)
+            cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m playwright install"
+            subprocess.run(cmdStr, shell=True)
             self.upgradeVenvThreadSignal.emit(True)
         except subprocess.CalledProcessError:
             self.upgradeVenvThreadSignal.emit(False)
@@ -834,7 +815,7 @@ class UpgradeVenvThread(QtCore.QThread):
 
 class InstallRequirementsThread(QtCore.QThread):
     progressSignal = QtCore.Signal(int)
-    installFinishedSignal = QtCore.Signal(bool, str)
+    installFinishedSignal = QtCore.Signal(bool, str, str, str)
 
     def __init__(self, modulesManager: ModulesManager, sourceUUID: str, modulePackUUID: str):
         super().__init__()
@@ -848,14 +829,7 @@ class InstallRequirementsThread(QtCore.QThread):
 
         if packDetails['Installed']:
             self.progressSignal.emit(3)
-            self.installFinishedSignal.emit(False, modulePackLabel)
+            self.installFinishedSignal.emit(False, modulePackLabel, "", "")
-            return
-
-
-        if (requirementsDetails :=
-        self.modulesManager.loadYamlFile(self.modulesManager.modulesRequirementsTrackingPath)) is None:
-            self.progressSignal.emit(3)
-            self.installFinishedSignal.emit(False, modulePackLabel)
             return
 
         requirementsSet = set()
@@ -865,11 +839,8 @@ class InstallRequirementsThread(QtCore.QThread):
             with open(moduleRequirements, 'r') as file:
                 [requirementsSet.add(line) for line in file.read().splitlines() if not line.startswith('#')]
 
-        moduleRequirements = list(requirementsSet)
+        with open(self.modulesManager.modulesRequirementsPath, 'r') as reqFile:
-        requirementsDetails[self.modulePackUUID] = moduleRequirements
+            [requirementsSet.add(line) for line in reqFile.read().splitlines() if not line.startswith('#')]
-
-        for module, requirements in requirementsDetails.items():
-            [requirementsSet.add(line) for line in requirements]
 
         with open(self.modulesManager.modulesRequirementsTempPath, 'w') as file:
             [file.write(line + '\n') for line in requirementsSet]
@@ -882,9 +853,6 @@ class InstallRequirementsThread(QtCore.QThread):
         try:
             subprocess.check_output(cmdStr, shell=True)
 
-            with open(self.modulesManager.modulesRequirementsTrackingPath, 'w') as file:
-                yaml.dump(requirementsDetails, file, default_flow_style=False)
-
             shutil.move(self.modulesManager.modulesRequirementsTempPath, self.modulesManager.modulesRequirementsPath)
             success = True
         except subprocess.CalledProcessError as cpe:
@@ -897,9 +865,69 @@ class InstallRequirementsThread(QtCore.QThread):
 
         if success:
             self.modulesManager.modulePacks[self.sourceUUID][self.modulePackUUID]['Installed'] = True
-            self.installFinishedSignal.emit(True, modulePackLabel)
+            self.installFinishedSignal.emit(True, modulePackLabel, self.sourceUUID, self.modulePackUUID)
         else:
-            self.installFinishedSignal.emit(False, modulePackLabel)
+            self.installFinishedSignal.emit(False, modulePackLabel, "", "")
+
+
+class UninstallModuleThread(QtCore.QThread):
+    progressSignal = QtCore.Signal(int)
+    uninstallFinishedSignal = QtCore.Signal(bool, str)
+
+    def __init__(self, modulesManager: ModulesManager, modulePackDict: dict):
+        super().__init__()
+        self.modulesManager = modulesManager
+        self.modulePackDict = modulePackDict
+
+    def run(self) -> None:
+        modulePackLabel = self.modulePackDict['label']
+        modulePackSourceUUID = self.modulePackDict['Source UUID']
+        modulePackUUID = self.modulePackDict['UUID']
+
+        if not self.modulePackDict['Installed']:
+            self.progressSignal.emit(3)
+            self.uninstallFinishedSignal.emit(False, modulePackLabel)
+            return
+
+        newRequirementsSet = set()
+        for sourceDict in self.modulesManager.sources.values():
+            sourceUUID = sourceDict['UUID']
+            for modulePackDetails in self.modulesManager.modulePacks[sourceUUID].values():
+                if modulePackUUID == modulePackDetails['UUID'] and \
+                            modulePackSourceUUID == modulePackDetails['Source UUID']:
+                    pass
+                elif modulePackDetails['Installed']:
+                    for module in modulePackDetails['modules']:
+                        moduleReqsPath = self.modulesManager.modulesBaseDirectoryPath / sourceUUID / module / 'requirements.txt'
+                        with open(moduleReqsPath, 'r') as file:
+                            [newRequirementsSet.add(line) for line in file.read().splitlines()
+                             if not line.startswith('#')]
+
+        with open(self.modulesManager.modulesRequirementsPath, 'r') as reqsFile:
+            allRequirementsSet = {line for line in reqsFile.read().splitlines() if not line.startswith('#')}
+
+        with open(self.modulesManager.modulesRequirementsTempPath, 'w') as file:
+            [file.write(line + '\n') for line in newRequirementsSet]
+
+        if reqsDiff := allRequirementsSet.difference(newRequirementsSet):
+            cmdStr = f"'{self.modulesManager.modulesPythonPath}' -m pip uninstall {' '.join(reqsDiff)} -y"
+            try:
+                subprocess.check_output(cmdStr, shell=True)
+                shutil.move(self.modulesManager.modulesRequirementsTempPath,
+                            self.modulesManager.modulesRequirementsPath)
+            except subprocess.CalledProcessError as cpe:
+                self.modulesManager.mainWindow.errorSignalListener.emit(
+                    f'Error occurred while uninstalling Module Pack "{modulePackLabel}":\n{cpe}', True, False)
+                self.modulesManager.modulesRequirementsTempPath.unlink(missing_ok=True)
+                self.progressSignal.emit(3)
+                self.uninstallFinishedSignal.emit(True, modulePackLabel)
+                return
+
+        self.modulesManager.modulePacks[modulePackSourceUUID][modulePackUUID]['Installed'] = False
+
+        self.progressSignal.emit(3)
+        self.uninstallFinishedSignal.emit(True, modulePackLabel)
+
 
 class AuthType(int, Enum):
     NONE = 0

Core/ResourceHandler.py

@@ -111,7 +111,7 @@ class ResourceHandler:
             'ISINID': re.compile(r"""^[a-zA-Z0-9]{2}-?[a-zA-Z0-9]{9}-?[a-zA-Z0-9]$"""),
             'SIC/NAICS': re.compile(r"""^[0-9]{4,6}$""")}
 
-        self.loadModuleEntities(self.programBaseDirPath / "Core" / "Entities")
+        self.loadModuleEntities(self.programBaseDirPath / "Core")
 
     def getPictureFromFile(self, filePath: Path, resize: tuple = (0, 0)) -> Optional[QByteArray]:
         """
@@ -146,7 +146,9 @@ class ResourceHandler:
         return pictureByteArray
 
     def loadModuleBanners(self, modulePath: Path):
-        pass  # TODO
+        assetsPath = modulePath / "Assets"
+        self.banners.update({f"{bannerPath.split('Banner_')[-1].split('.')[0]}": str(bannerPath)
+                             for bannerPath in glob(str(assetsPath / "Banner_*.svg"))})
 
     def getEntityCategories(self) -> list:
         return list(self.entityCategoryList)
@@ -285,13 +287,17 @@ class ResourceHandler:
         return entityTypesAdded
 
     def loadModuleEntities(self, modulePath: Path) -> list:
+        entitiesPath = modulePath / 'Entities'
         allModuleEntitiesAdded = []
-        self.moduleAssetPaths.append(modulePath / "assets")
+        for entFile in listdir(entitiesPath):
-        for entFile in listdir(modulePath):
             if entFile.endswith('.xml'):
-                allModuleEntitiesAdded += self.addRecognisedEntityTypes(modulePath / entFile)
+                allModuleEntitiesAdded += self.addRecognisedEntityTypes(entitiesPath / entFile)
         return allModuleEntitiesAdded
 
+    def loadModuleAssets(self, modulePath: Path):
+        self.moduleAssetPaths.append(modulePath / "Assets")
+        self.loadModuleBanners(modulePath)
+
     def getEntityJson(self, entityType: str, jsonData=None) -> Union[dict, None]:
         eJson = {'uid': str(uuid4())}
         if entityType in self.getAllEntitiesInCategory('Meta'):

LinkScope.py

@@ -931,13 +931,14 @@ class MainWindow(QtWidgets.QMainWindow):
         self.MODULEMANAGER.loadAllModules()
         self.setStatus('Loaded Modules.')
 
-    def reloadModules(self) -> None:
+    def reloadModules(self, onlyUpdateDockbar:bool = False) -> None:
         """
         Same as loadModules, except this one updates the GUI to show newly loaded entities and resolutions.
         This is meant to be run after the application started, in case the user wants to load a module without
         closing and reopening the application.
         :return:
         """
+        if not onlyUpdateDockbar:
             self.MODULEMANAGER.loadAllModules()
         self.dockbarOne.existingEntitiesPalette.loadEntities()
         self.dockbarOne.resolutionsPalette.loadAllResolutions()
@@ -1888,7 +1889,6 @@ class MainWindow(QtWidgets.QMainWindow):
         if len(self.centralWidget().tabbedPane.canvasTabs) == 0:
             self.centralWidget().tabbedPane.createHomeTab()
 
-        self.setStatus("Ready")
         self.MESSAGEHANDLER.info(f"Project {self.SETTINGS.value('Project/Name', 'Untitled')} opened.")
 
         if self.SETTINGS.value("Program/Usage/First Time Start", 'true') == 'true':
@@ -2334,8 +2334,8 @@ class EntityPage(QtWidgets.QWizardPage):
         else:
             if 'svg' not in str(self.defaultpic):
                 # Default picture is an SVG.
-                self.defaultpic = self.parent().RESOURCEHANDLER.getEntityDefaultPicture(
+                self.defaultpic = self.reportWizard.parent().RESOURCEHANDLER.getEntityDefaultPicture(
-                    self.parent().LENTDB.getEntity(self.entityUID)['Entity Type'])
+                    self.reportWizard.parent().LENTDB.getEntity(self.entityUID)['Entity Type'])
             contents = bytearray(self.defaultpic)
             widthRegex = re.compile(b' width="\d*" ')
             fileContents = ''