Ianhelle/msticpy 3.0 and Python 3.13

.github/workflows/python-package.yml

@@ -17,7 +17,7 @@ jobs:
     permissions: read-all
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
       # Print out details about the run
       - name: Dump GitHub context
@@ -178,18 +178,11 @@ jobs:
             python -m pip install -r requirements-dev.txt
           else
             echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
-            python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
+            python -m pip install bandit mypy types-attrs pyroma
           fi
-      - name: black
+      - name: ruff
         run: |
-          black --diff --check --exclude venv msticpy
-        if: ${{ always() }}
-      - name: flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 msticpy --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+          ruff check msticpy --ignore PLW0603
         if: ${{ always() }}
       - name: pylint
         run: |
@@ -221,14 +214,6 @@ jobs:
         run: |
           bandit -x tests -r -s B303,B404,B603,B607,B608,B113 msticpy
         if: ${{ always() }}
-      - name: flake8
-        run: |
-          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
-        if: ${{ always() }}
-      - name: pydocstyle
-        run: |
-          pydocstyle --convention=numpy msticpy
-        if: ${{ always() }}
       - name: pyroma
         run: |
           pyroma --min 10 .

.pre-commit-config.yaml

@@ -7,34 +7,13 @@ repos:
         exclude: .*devcontainer.json
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
-  - repo: https://github.com/ambv/black
-    rev: 25.9.0
-    hooks:
-      - id: black
-        language: python
   - repo: https://github.com/PyCQA/pylint
     rev: v4.0.2
     hooks:
       - id: pylint
         args:
           - --disable=duplicate-code,import-error
           - --ignore-patterns=test_
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.3.0
-    hooks:
-      - id: flake8
-        args:
-          - --extend-ignore=E401,E501,W503
-          - --max-line-length=90
-          - --exclude=tests,test*.py
-  - repo: https://github.com/pycqa/isort
-    rev: 7.0.0
-    hooks:
-      - id: isort
-        name: isort (python)
-        args:
-          - --profile
-          - black
   - repo: https://github.com/pycqa/pydocstyle
     rev: 6.3.0
     hooks:
@@ -43,14 +22,17 @@ repos:
           - --convention=numpy
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.14.1
+    rev: v0.14.8
     hooks:
       # Run the linter.
-      - id: ruff
-        types_or: [ python, pyi, jupyter ]
+      - id: ruff-check
         args:
         - msticpy
         - --fix
+      # Run the formatter.
+      - id: ruff-format
+        args:
+        - msticpy
   - repo: local
     hooks:
       - id: check_reqs_all

README.md

@@ -3,7 +3,6 @@
 ![GitHub Actions build](https://github.com/microsoft/msticpy/actions/workflows/python-package.yml/badge.svg?branch=main)
 [![Azure Pipelines build](https://dev.azure.com/mstic-detections/mstic-jupyter/_apis/build/status/microsoft.msticpy?branchName=main)](https://dev.azure.com/mstic-detections/mstic-jupyter/_build/latest?definitionId=14&branchName=main)
 [![Downloads](https://pepy.tech/badge/msticpy)](https://pepy.tech/project/msticpy)
-[![BlackHat Arsenal 2020](https://raw.githubusercontent.com/toolswatch/badges/master/arsenal/usa/2020.svg)](https://www.blackhat.com/us-20/arsenal/schedule/#msticpy-the-security-analysis-swiss-army-knife-19872)
 
 Microsoft Threat Intelligence Python Security Tools.
 
@@ -29,7 +28,7 @@ alt="Timeline" title="Msticpy Timeline Control" height="300" />
 The **msticpy** package was initially developed to support
 [Jupyter Notebooks](https://jupyter-notebook-beginner-guide.readthedocs.io/en/latest/)
 authoring for
-[Azure Sentinel](https://azure.microsoft.com/en-us/services/azure-sentinel/).
+[Microsoft Sentinel](https://www.microsoft.com/en-us/security/business/siem-and-xdr/microsoft-sentinel/).
 While Azure Sentinel is still a big focus of our work, we are
 extending the data query/acquisition components to pull log data from
 other sources (currently Splunk, Microsoft Defender for Endpoint and
@@ -55,11 +54,6 @@ For core install:
 
 `pip install msticpy`
 
-If you are using *MSTICPy* with Azure Sentinel you should install with
-the "azsentinel" extra package:
-
-`pip install msticpy[azsentinel]`
-
 or for the latest dev build
 
 `pip install git+https://github.com/microsoft/msticpy`
@@ -90,8 +84,8 @@ functions in this interactive demo on mybinder.org.
 
 ## Log Data Acquisition
 
-QueryProvider is an extensible query library targeting Azure Sentinel/Log Analytics,
-Splunk, OData
+QueryProvider is an extensible query library targeting Microsoft Sentinel/Log Analytics,
+Microsoft XDR, Splunk, OData
 and other log data sources. It also has special support for
 [Mordor](https://github.com/OTRF/mordor) data sets and using local data.
 
@@ -325,7 +319,7 @@ See the following notebooks for more examples of the use of this package in prac
 ## Supported Platforms and Packages
 
 - msticpy is OS-independent
-- Requires [Python 3.8 or later](https://www.python.org/dev/peps/pep-0494/)
+- Requires [Python 3.10 or later](https://www.python.org/dev/peps/pep-0494/)
 - See [requirements.txt](requirements.txt) for more details and version requirements.
 
 ---

conda/conda-reqs-dev.txt

@@ -4,18 +4,14 @@ beautifulsoup4
 black>=20.8b1
 coverage>=5.5
 filelock>=3.0.0
-flake8>=3.8.4
-isort>=5.10.1
 markdown>=3.3.4
-mccabe>=0.6.1
 mypy>=0.821
 nbconvert>=6.1.0
 nbdime>=2.1.0
 pandas>=1.4.0
 pep8-naming>=0.10.0
 pep8>=1.7.1
 pipreqs>=0.4.9
-pycodestyle>=2.6.0
 pydocstyle>=6.0.0
 pyflakes>=2.2.0
 pylint>=2.5.3

conda/conda-reqs-pip.txt

@@ -1,13 +1,11 @@
 azure-mgmt-resourcegraph>=8.0.0
 azure-monitor-query>=1.0.0, <=2.0.0
-# KqlmagicCustom[jupyter-basic,auth_code_clipboard]>=0.1.114.post22
 mo-sql-parsing>=11, <12.0.0
 nest_asyncio>=1.4.0
 passivetotal>=2.5.3
 sumologic-sdk>=0.1.11
 splunk-sdk>=1.6.0,!=2.0.0
 packaging>=24.0
 requests>=2.31.0
-importlib-resources >= 6.4.0; python_version <= "3.8"
 rrcf==0.4.4
 joblib>=1.3.0

docs/source/api/msticpy.data.drivers.prismacloud_driver.rst

@@ -0,0 +1,7 @@
+msticpy.data.drivers.prismacloud\_driver module
+===============================================
+
+.. automodule:: msticpy.data.drivers.prismacloud_driver
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/msticpy.data.drivers.rst

@@ -26,6 +26,7 @@ Submodules
    msticpy.data.drivers.mdatp_driver
    msticpy.data.drivers.mordor_driver
    msticpy.data.drivers.odata_driver
+   msticpy.data.drivers.prismacloud_driver
    msticpy.data.drivers.resource_graph_driver
    msticpy.data.drivers.security_graph_driver
    msticpy.data.drivers.sentinel_query_reader

docs/source/conf.py

@@ -257,7 +257,6 @@
     "ipywidgets",
     "jwt",
     "keyring",
-    "Kqlmagic",
     "matplotlib.pyplot",
     "matplotlib",
     "mo_sql_parsing",

msticpy/__init__.py

@@ -114,8 +114,8 @@
 initialization and checks are performed.
 
 """
-import os
-from typing import Iterable, Union
+
+from collections.abc import Iterable
 
 from . import nbwidgets  # noqa: F401
 
@@ -134,9 +134,6 @@
 get_config = settings.get_config
 setup_logging()
 
-if not os.environ.get("KQLMAGIC_EXTRAS_REQUIRES"):
-    os.environ["KQLMAGIC_EXTRAS_REQUIRES"] = "jupyter-basic"
-
 _LAZY_IMPORTS = {
     "msticpy.auth.azure_auth.az_connect",
     "msticpy.common.timespan.TimeSpan",
@@ -159,7 +156,7 @@
 module, __getattr__, __dir__ = lazy_import(__name__, _LAZY_IMPORTS)
 
 
-def load_plugins(plugin_paths: Union[str, Iterable[str]]):
+def load_plugins(plugin_paths: str | Iterable[str]):
     """
     Load plugins from specified paths or configuration.
 
@@ -177,6 +174,6 @@ def load_plugins(plugin_paths: Union[str, Iterable[str]]):
 
     """
     # pylint: disable=import-outside-toplevel
-    from .init.mp_plugins import read_plugins
+    from .init.mp_plugins import read_plugins  # noqa: PLC0415
 
     read_plugins(plugin_paths)

msticpy/_version.py

@@ -1,3 +1,3 @@
 """Version file."""
 
-VERSION = "2.18.0"
+VERSION = "3.0.0.pre1"

msticpy/analysis/anomalous_sequence/anomalous.py

@@ -8,6 +8,7 @@
 
 In particular, this module is for both modelling and visualising your session data.
 """
+
 from __future__ import annotations
 
 import pandas as pd
@@ -61,17 +62,17 @@ def score_sessions(
         raise MsticpyException(f'"{session_column}" should be a column in the `data`')
 
     sessions_df = data.copy()
-    sessions = sessions_df[session_column].values.tolist()  # type: ignore
+    sessions = sessions_df[session_column].values.tolist()
 
     model = Model(sessions=sessions)
     model.train()
     model.compute_rarest_windows(
         window_len=window_length, use_geo_mean=False, use_start_end_tokens=True
     )
 
-    sessions_df[f"rarest_window{window_length}_likelihood"] = (
-        model.rare_window_likelihoods[window_length]
-    )
+    sessions_df[f"rarest_window{window_length}_likelihood"] = model.rare_window_likelihoods[
+        window_length
+    ]
     sessions_df[f"rarest_window{window_length}"] = model.rare_windows[window_length]
 
     return sessions_df

msticpy/analysis/anomalous_sequence/model.py

@@ -6,7 +6,6 @@
 """Module for Model class for modelling sessions data."""
 
 from collections import defaultdict
-from typing import Dict, List, Union
 
 from ...common.exceptions import MsticpyException
 from .utils import cmds_only, cmds_params_only, cmds_params_values, probabilities
@@ -18,9 +17,7 @@
 class Model:
     """Class for modelling sessions data."""
 
-    def __init__(
-        self, sessions: List[List[Union[str, Cmd]]], modellable_params: set = None
-    ):
+    def __init__(self, sessions: list[list[str | Cmd]], modellable_params: set = None):
         """
         Instantiate the Model class.
 
@@ -105,16 +102,16 @@ def __init__(
         self.value_probs = None
         self.value_cond_param_probs = None
 
-        self.set_params_cond_cmd_probs: Dict[str, Dict[str, float]] = {}
+        self.set_params_cond_cmd_probs: dict[str, dict[str, float]] = {}
 
         self.session_likelihoods = None
         self.session_geomean_likelihoods = None
 
-        self.rare_windows: Dict[int, list] = {}
-        self.rare_window_likelihoods: Dict[int, list] = {}
+        self.rare_windows: dict[int, list] = {}
+        self.rare_window_likelihoods: dict[int, list] = {}
 
-        self.rare_windows_geo: Dict[int, list] = {}
-        self.rare_window_likelihoods_geo: Dict[int, list] = {}
+        self.rare_windows_geo: dict[int, list] = {}
+        self.rare_window_likelihoods_geo: dict[int, list] = {}
 
     def train(self):
         """
@@ -154,9 +151,7 @@ def compute_scores(self, use_start_end_tokens: bool):
 
         """
         if self.prior_probs is None:
-            raise MsticpyException(
-                "please train the model first before using this method"
-            )
+            raise MsticpyException("please train the model first before using this method")
         self.compute_likelihoods_of_sessions(use_start_end_tokens=use_start_end_tokens)
         self.compute_geomean_lik_of_sessions()
         self.compute_rarest_windows(
@@ -339,7 +334,7 @@ def _compute_probs(self):
         if self.session_type == SessionType.cmds_params_values:
             self._compute_probs_values()
 
-    def compute_setof_params_cond_cmd(self, use_geo_mean: bool):  # noqa: MC0001
+    def compute_setof_params_cond_cmd(self, use_geo_mean: bool):
         """
         Compute likelihood of combinations of params conditional on the cmd.
 
@@ -370,9 +365,7 @@ def compute_setof_params_cond_cmd(self, use_geo_mean: bool):  # noqa: MC0001
 
         """
         if self.param_probs is None:
-            raise MsticpyException(
-                "please train the model first before using this method"
-            )
+            raise MsticpyException("please train the model first before using this method")
 
         if self.session_type is None:
             raise MsticpyException("session_type attribute should not be None")
@@ -442,9 +435,7 @@ def compute_likelihoods_of_sessions(self, use_start_end_tokens: bool = True):
 
         """
         if self.prior_probs is None:
-            raise MsticpyException(
-                "please train the model first before using this method"
-            )
+            raise MsticpyException("please train the model first before using this method")
 
         result = []
 
@@ -556,9 +547,7 @@ def compute_rarest_windows(
 
         """
         if self.prior_probs is None:
-            raise MsticpyException(
-                "please train the model first before using this method"
-            )
+            raise MsticpyException("please train the model first before using this method")
 
         if self.session_type == SessionType.cmds_only:
             rare_tuples = [
@@ -609,9 +598,7 @@ def compute_rarest_windows(
 
         if use_geo_mean:
             self.rare_windows_geo[window_len] = [rare[0] for rare in rare_tuples]
-            self.rare_window_likelihoods_geo[window_len] = [
-                rare[1] for rare in rare_tuples
-            ]
+            self.rare_window_likelihoods_geo[window_len] = [rare[1] for rare in rare_tuples]
         else:
             self.rare_windows[window_len] = [rare[0] for rare in rare_tuples]
             self.rare_window_likelihoods[window_len] = [rare[1] for rare in rare_tuples]

msticpy/analysis/anomalous_sequence/sessionize.py

@@ -4,6 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 """Module for creating sessions out of raw data."""
+
 from __future__ import annotations
 
 import numpy as np