From 9a6335a6a1d3e31612eab942e7125f8ef33b4f6d Mon Sep 17 00:00:00 2001
From: ErikMangstenRecFut <erik.mangsten@recordedfuture.com>
Date: Mon, 27 Jan 2025 11:09:26 +0100
Subject: [PATCH] remove recorded future sigma rules importer

This example is outdated, unused and broken.
---
 ...Recorded Future Sigma Rules Importer.ipynb | 820 ------------------
 1 file changed, 820 deletions(-)
 delete mode 100644 tutorials-and-examples/example-notebooks/Recorded Future Sigma Rules Importer.ipynb

diff --git a/tutorials-and-examples/example-notebooks/Recorded Future Sigma Rules Importer.ipynb b/tutorials-and-examples/example-notebooks/Recorded Future Sigma Rules Importer.ipynb
deleted file mode 100644
index 36203758..00000000
--- a/tutorials-and-examples/example-notebooks/Recorded Future Sigma Rules Importer.ipynb	
+++ /dev/null
@@ -1,820 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "source": [
-        "# Import and convert Recorded Future Sigma rules\n",
-        "Created by Jonah Feldman, Recorded Future. Heavily based off of [this](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/c89c9156ab26092efe7365d37e9e97fe4b7571e1/tutorials-and-examples/example-notebooks/SigmaRuleImporter.ipynb) notebook by ianhelle@microsoft.com\n",
-        "\n",
-        "**NOTE:** This is a beta version. We are constantly improving its features and functionality to provide you with the best user experience possible. Your feedback is valuable and helps us identify areas of improvement. Please send any suggestions or issues to int-saas-team@recordedfuture.com.\n",
-        "\n",
-        "This notebook fetches sigma rules created by Recorded Future's Insikt team and converts them to KQL. After conversion, you have the option to interactively query your Log Analytics workspace with these rules or create a Sentine Analytic rule to generate alerts/incidents based off it's detections\n",
-        "\n",
-        "\n",
-        "Known issues:\n",
-        "- Authentication to Sentinel can sometimes fail to detect msticpyconfig.yaml even when it's present. As a workaround, you can directly hardcode the, subscription id, resource group name, and workspace name into the cell\n",
-        "- Techniques and not currently translated correctly to the Analytic Rule. This is being worked on\n",
-        "- Backslashes are not always escaped properly and some rules with backlashes in the query will not always execute correctly\n",
-        "- In certain shells, `msticpy[azure]` in the first cell needs to be wrapped in quotes\n",
-        "\n",
-        "By Default, all Analytic rules are create disabled"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Install dependencies"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from pathlib import Path\n",
-        "from IPython.display import display, HTML\n",
-        "\n",
-        "REQ_PYTHON_VER = \"3.6\"\n",
-        "REQ_MSTICPY_VER = \"1.0.0\"\n",
-        "\n",
-        "display(HTML(\"<h3>Starting Notebook setup...</h3>\"))\n",
-        "\n",
-        "# If not using Azure Notebooks, install msticpy with\n",
-        "#%pip install msticpy\n",
-        "#%pip install msticpy[azure] #OR %pip install \"msticpy[azure]\"\n",
-        "#%pip install sigmatools\n",
-        "from msticpy.nbtools import nbinit\n",
-        "nbinit.init_notebook(namespace=globals());"
-      ],
-      "outputs": [
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<h3>Starting Notebook setup...</h3>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<h4>Starting notebook pre-checks...</h4>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "Checking Python kernel version..."
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "Info: Python kernel version 3.8.10 - OK<br>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "Checking msticpy version...<br>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "Info: msticpy version 2.3.1 (>= 1.0.0) - OK<br>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<h4>Notebook pre-checks complete.</h4>"
-          },
-          "metadata": {}
-        }
-      ],
-      "execution_count": 1,
-      "metadata": {
-        "gather": {
-          "logged": 1683621776020
-        },
-        "jupyter": {
-          "outputs_hidden": false,
-          "source_hidden": false
-        },
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Enter your Recorded Future API key\n",
-        "This notebook requires an API key to communicate with the Recorded Future API. To obtain API keys, please visit [Recorded Future Requesting API Tokens](https://support.recordedfuture.com/hc/en-us/articles/4411077373587-Requesting-API-Tokens)."
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import getpass\n",
-        "\n",
-        "rf_token = getpass.getpass()"
-      ],
-      "outputs": [],
-      "execution_count": 2,
-      "metadata": {
-        "scrolled": true,
-        "gather": {
-          "logged": 1683619357285
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Fetch Rules from the Recorded Future API. This may take awhile\n",
-        "Modify query here. E.g. time interval (after, before), number of retrieved Sigma rules (limit). For more inspiration check out the API specification [here](https://api.recordedfuture.com/openapi/detection-rule.html)."
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import requests\n",
-        "import datetime\n",
-        "\n",
-        "# Download Sigma rules\n",
-        "# Modify the query below as needed, Tip reduce number of days if receiving \"Output exceeds the size limit.\"\n",
-        "from_date = (datetime.datetime.utcnow() - datetime.timedelta(days=180)).isoformat()\n",
-        "to_date = datetime.datetime.utcnow().isoformat()\n",
-        "query = {\n",
-        "    \"filter\": {\n",
-        "        \"types\": [\n",
-        "            \"sigma\"\n",
-        "        ],\n",
-        "        \"created\": {\n",
-        "            \"after\": from_date,\n",
-        "            \"before\": to_date\n",
-        "        }\n",
-        "    },\n",
-        "    \"tagged_entities\": False,\n",
-        "    \"limit\": 100\n",
-        "}\n",
-        "rf_detection_api_url = 'https://api.recordedfuture.com/detection-rule/search'\n",
-        "r = requests.post(rf_detection_api_url, headers={'X-RFToken': rf_token}, json=query)\n",
-        "r.raise_for_status()\n",
-        "display(f'Retrieved {r.json()[\"count\"]} Sigma Rules')\n",
-        "display(f'Fromdate {from_date} todate {to_date}')"
-      ],
-      "outputs": [
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "'Retrieved 21 Sigma Rules'"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "'Fromdate 2022-11-10T08:02:37.219719 todate 2023-05-09T08:02:37.219755'"
-          },
-          "metadata": {}
-        }
-      ],
-      "execution_count": 3,
-      "metadata": {
-        "gather": {
-          "logged": 1683619361149
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Save the rules locally. "
-      ],
-      "metadata": {
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import yaml\n",
-        "from ipywidgets import widgets, Layout\n",
-        "import os\n",
-        "from pathlib import Path\n",
-        "import traceback\n",
-        "\n",
-        "def_path = Path.joinpath(Path(os.getcwd()), \"rf-sigma\")\n",
-        "path_wgt = widgets.Text(value=str(def_path), \n",
-        "                        description='Path to extract to zipped repo files: ', \n",
-        "                        layout=Layout(width='50%'),\n",
-        "                        style={'description_width': 'initial'})\n",
-        "path_wgt\n",
-        "rules_root = Path(path_wgt.value)\n",
-        "try:\n",
-        "    os.mkdir(rules_root)\n",
-        "except FileExistsError as err:\n",
-        "    pass\n",
-        "\n",
-        "for result in r.json()['result']:\n",
-        "    for rule in result['rules']:\n",
-        "        content = rule['content']\n",
-        "        title = list(yaml.safe_load_all(content))[0]['title'] + '.yml'\n",
-        "        with open(Path.joinpath(rules_root, title), 'w') as file:\n",
-        "            file.write(content)\n"
-      ],
-      "outputs": [],
-      "execution_count": 4,
-      "metadata": {
-        "gather": {
-          "logged": 1683619362973
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Check that we have the files\n",
-        "You should see a folder with folders such as application, apt, windows..."
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "%ls {rules_root}"
-      ],
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": " \u001b[0m\u001b[01;32mAPT_RU_Gamaredon_SFX_Loader.yml\u001b[0m*\r\n \u001b[01;32mAPT_RU_Gamaredon_SFX_Loader_cleanup.yml\u001b[0m*\r\n\u001b[01;32m'Bitsadmin Command to Download File from Web.yml'\u001b[0m*\r\n\u001b[01;32m'Commodity Packer Processes.yml'\u001b[0m*\r\n\u001b[01;32m'MAL Remcos RAT.yml'\u001b[0m*\r\n \u001b[01;32mMAL_ASYNCRAT_DNS_EVENTS.yml\u001b[0m*\r\n \u001b[01;32mMAL_ASYNCRAT_POWERSHELL_FILECREATION.yml\u001b[0m*\r\n \u001b[01;32mMAL_AcridRain.yml\u001b[0m*\r\n \u001b[01;32mMAL_Aesthetic_Wiper.yml\u001b[0m*\r\n \u001b[01;32mMAL_Cuba_Ransomware.yml\u001b[0m*\r\n \u001b[01;32mMAL_DarkOwl_Download_Payload.yml\u001b[0m*\r\n \u001b[01;32mMAL_DivergentGap.yml\u001b[0m*\r\n \u001b[01;32mMAL_Kodex_Ransomware.yml\u001b[0m*\r\n \u001b[01;32mMAL_LockBit3_Shadowcopy_Deletion.yml\u001b[0m*\r\n \u001b[01;32mMAL_Lorenz_Ransomware.yml\u001b[0m*\r\n \u001b[01;32mMAL_MagicRAT.yml\u001b[0m*\r\n \u001b[01;32mMAL_NullMixer.yml\u001b[0m*\r\n \u001b[01;32mMAL_PWNKIT_Exploit_File_Event.yml\u001b[0m*\r\n \u001b[01;32mMAL_Qakbot.yml\u001b[0m*\r\n \u001b[01;32mMAL_SharpHound_process.yml\u001b[0m*\r\n \u001b[01;32mMAL_Vidar_Stealer.yml\u001b[0m*\r\n \u001b[01;32mMAL_disable_modify_tools.yml\u001b[0m*\r\n \u001b[01;32mMal_Trochilus_Interaction.yml\u001b[0m*\r\n\u001b[01;32m'Permissions Modification.yml'\u001b[0m*\r\n\u001b[01;32m'Startup Folder Modification in Registry.yml'\u001b[0m*\r\n\u001b[01;32m'XMRig Mining Software - Processes Created.yml'\u001b[0m*\r\n"
-        }
-      ],
-      "execution_count": 5,
-      "metadata": {
-        "scrolled": true
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Convert Sigma Files to Log Analytics KQL queries"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "# Sigma to Log Analytics Conversion\n",
-        "import yaml\n",
-        "import os\n",
-        "import sys\n",
-        "module_path = os.path.abspath(os.path.join('sigma/sigma-master/tools'))\n",
-        "if module_path not in sys.path:\n",
-        "    sys.path.append(module_path)\n",
-        "from sigma.parser.collection import SigmaCollectionParser\n",
-        "from sigma.parser.exceptions import SigmaCollectionParseError, SigmaParseError\n",
-        "from sigma.configuration import SigmaConfiguration, SigmaConfigurationChain\n",
-        "from sigma.config.exceptions import SigmaConfigParseError, SigmaRuleFilterParseException\n",
-        "from sigma.filter import SigmaRuleFilter\n",
-        "import sigma.backends.discovery as backends\n",
-        "from sigma.backends.base import BackendOptions\n",
-        "from sigma.backends.exceptions import BackendError, NotSupportedError, PartialMatchError, FullMatchError\n",
-        "from pathlib import Path\n",
-        "\n",
-        "    \n",
-        "sigma_list = list(Path(rules_root).resolve().glob(\"*.yml\"))\n",
-        "\n",
-        "_LA_MAPPINGS = '''\n",
-        "fieldmappings:\n",
-        "  Image: NewProcessName\n",
-        "  ParentImage: ParentProcessName\n",
-        "  ParentCommandLine: NO_MAPPING\n",
-        "'''\n",
-        "\n",
-        "NOT_CONVERTIBLE = 'Not convertible'\n",
-        "\n",
-        "def sigma_to_la(file_path):\n",
-        "    with open(file_path, 'r') as input_file:\n",
-        "        try:\n",
-        "            sigmaconfigs = SigmaConfigurationChain()\n",
-        "            sigmaconfig = SigmaConfiguration(_LA_MAPPINGS)\n",
-        "            sigmaconfigs.append(sigmaconfig)\n",
-        "            backend_options = BackendOptions(None, None)\n",
-        "            backend = backends.getBackend('ala')(sigmaconfigs, backend_options)\n",
-        "            parser = SigmaCollectionParser(input_file, sigmaconfigs, None)\n",
-        "            results = parser.generate(backend)\n",
-        "            kql_result = ''\n",
-        "            for result in results:\n",
-        "                kql_result += result\n",
-        "        except (NotImplementedError, NotSupportedError, TypeError):\n",
-        "            kql_result = NOT_CONVERTIBLE\n",
-        "        input_file.seek(0,0)\n",
-        "        sigma_txt = input_file.read()\n",
-        "        if not kql_result == NOT_CONVERTIBLE:\n",
-        "            try:\n",
-        "                kql_header = \"\\n\".join(get_sigma_properties(sigma_txt))\n",
-        "                kql_result = kql_header + \"\\n\" + kql_result\n",
-        "            except Exception as e:\n",
-        "                print(\"exception reading sigma YAML: \", e)\n",
-        "                print(sigma_txt, kql_result, sep='\\n')\n",
-        "        return sigma_txt, kql_result\n",
-        "\n",
-        "sigma_keys = ['title', 'description', 'tags', 'status', \n",
-        "              'author', 'logsource', 'falsepositives', 'level']\n",
-        "\n",
-        "def get_sigma_properties(sigma_rule):\n",
-        "    sigma_docs = yaml.load_all(sigma_rule, Loader=yaml.SafeLoader)\n",
-        "    sigma_rule_dict = next(sigma_docs)\n",
-        "    for prop in sigma_keys:\n",
-        "        yield get_property(prop, sigma_rule_dict)\n",
-        "\n",
-        "def get_property(name, sigma_rule_dict):\n",
-        "    sig_prop = sigma_rule_dict.get(name, 'na')\n",
-        "    if isinstance(sig_prop, dict):\n",
-        "        sig_prop = ' '.join([f\"{k}: {v}\" for k, v in sig_prop.items()])\n",
-        "    return f\"// {name}: {sig_prop}\"\n",
-        "        \n",
-        "   \n",
-        "_KQL_FILTERS = {\n",
-        "    'date': ' | where TimeGenerated >= datetime({start}) and TimeGenerated <= datetime({end}) ',\n",
-        "    'host': ' | where Computer has {host_name} '\n",
-        "}\n",
-        "\n",
-        "def insert_at(source, insert, find_sub):\n",
-        "    pos = source.find(find_sub)\n",
-        "    if pos != -1:\n",
-        "        return source[:pos] + insert + source[pos:]\n",
-        "    else:\n",
-        "        return source + insert\n",
-        "        \n",
-        "def add_filter_clauses(source, **kwargs):\n",
-        "    if \"{\" in source or \"}\" in source:\n",
-        "        source = (\"// Warning: embedded braces in source. Please edit if necessary.\\n\"\n",
-        "                  + source)\n",
-        "        source = source.replace('{', '{{').replace('}', '}}')\n",
-        "    if kwargs.get('host', False):\n",
-        "        source = insert_at(source, _KQL_FILTERS['host'], '|')\n",
-        "    if kwargs.get('date', False):\n",
-        "        source = insert_at(source, _KQL_FILTERS['date'], '|')\n",
-        "    return source\n",
-        "\n",
-        "\n",
-        "# Run the conversion\n",
-        "print(\"Converting rules\")\n",
-        "conv_counter = []\n",
-        "kql_dict = {}\n",
-        "for file_path in sigma_list:\n",
-        "    file_name = os.path.basename(file_path)\n",
-        "    src_converted = 0\n",
-        "    try:\n",
-        "        sigma, kql = sigma_to_la(file_path)\n",
-        "    except:\n",
-        "        print(f\"Error converting {file_name} ({file_path})\")\n",
-        "        continue\n",
-        "    print(f'Converted {file_name}')\n",
-        "    kql_dict[file_name] = (sigma, kql)\n",
-        "    if not kql == NOT_CONVERTIBLE:\n",
-        "        src_converted += 1\n",
-        "\n",
-        "print(\"\\nConversion statistics\")\n",
-        "print(\"-\" * len(\"Conversion statistics\"))\n",
-        "print(f'Rules: {len(sigma_list)}, Converted: {len(kql_dict.items())}')"
-      ],
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "Converting rules\nConverted APT_RU_Gamaredon_SFX_Loader.yml\nConverted APT_RU_Gamaredon_SFX_Loader_cleanup.yml\nConverted Bitsadmin Command to Download File from Web.yml\nConverted Commodity Packer Processes.yml\nConverted MAL Remcos RAT.yml\nConverted MAL_AcridRain.yml\nConverted MAL_Aesthetic_Wiper.yml\nConverted MAL_ASYNCRAT_DNS_EVENTS.yml\nConverted MAL_ASYNCRAT_POWERSHELL_FILECREATION.yml\nConverted MAL_Cuba_Ransomware.yml\nConverted MAL_DarkOwl_Download_Payload.yml\nConverted MAL_disable_modify_tools.yml\nConverted MAL_DivergentGap.yml\nConverted MAL_Kodex_Ransomware.yml\nConverted MAL_LockBit3_Shadowcopy_Deletion.yml\nConverted MAL_Lorenz_Ransomware.yml\nConverted MAL_MagicRAT.yml\nConverted MAL_NullMixer.yml\nConverted MAL_PWNKIT_Exploit_File_Event.yml\nConverted MAL_Qakbot.yml\nConverted MAL_SharpHound_process.yml\nConverted Mal_Trochilus_Interaction.yml\nConverted MAL_Vidar_Stealer.yml\nConverted Permissions Modification.yml\nConverted Startup Folder Modification in Registry.yml\nConverted XMRig Mining Software - Processes Created.yml\n\nConversion statistics\n---------------------\nRules: 26, Converted: 26\n"
-        }
-      ],
-      "execution_count": 6,
-      "metadata": {
-        "gather": {
-          "logged": 1683619364292
-        },
-        "scrolled": false
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Authenticate to Log Analytics"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from IPython.display import display\n",
-        "from msticpy.data import QueryProvider\n",
-        "from msticpy.common.wsconfig import WorkspaceConfig\n",
-        "\n",
-        "ws_config = WorkspaceConfig()\n",
-        "qry_prov = QueryProvider(\"LogAnalytics\")\n",
-        "qry_prov.connect(ws_config.code_connect_str)\n"
-      ],
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "Please wait. Loading Kqlmagic extension..."
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.Javascript object>",
-            "application/javascript": "try {IPython.notebook.kernel.reconnect();} catch(err) {;}"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "done\ntry_azcli_login=True;enable_add_items_to_help=False\n{'try_azcli_login': 'True', 'enable_add_items_to_help': 'False'}\ntry_azcli_login=True;enable_add_items_to_help=False\nConnecting... "
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<!DOCTYPE html>\n                                <html><body>\n                                <input  id=\"kql_MagicCodeAuthInput\" type=\"text\" readonly style=\"font-weight: bold; border: none;\" size='9' value='EGPS5WLCS'>\n\n                                <script>\n                                function kql_MagicCopyCodeFunction() {\n                                    /* Get the text field */\n                                    var copyText = document.getElementById(\"kql_MagicCodeAuthInput\");\n\n                                    /* Select the text field */\n                                    copyText.select();\n\n                                    /* Copy the text inside the text field */\n                                    document.execCommand(\"copy\");\n\n                                    /* Alert the copied text */\n                                    // alert(\"Copied the text: \" + copyText.value);\n                                }\n                                kql_MagicCopyCodeFunction()\n                                </script>\n\n                                </body></html>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "Copy code to clipboard and authenticate here: https://microsoft.com/devicelogin\n"
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<!DOCTYPE html>\n                            <html><body><script>\n\n                                // close authentication window\n                                if (kql_MagicUserCodeAuthWindow && kql_MagicUserCodeAuthWindow.opener != null && !kql_MagicUserCodeAuthWindow.closed) {\n                                    kql_MagicUserCodeAuthWindow.close()\n                                }\n                                // TODO: make sure, you clear the right cell. BTW, not sure it is a must to do any clearing\n\n                                // clear output cell\n                                Jupyter.notebook.clear_output(Jupyter.notebook.get_selected_index())\n\n                                // TODO: if in run all mode, move to last cell, otherwise move to next cell\n                                // move to next cell\n\n                            </script></body></html>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "<IPython.core.display.HTML object>",
-            "text/html": "<!DOCTYPE html>\n            <html><body>\n            <div style=''>\n            \n            <button onclick=\"this.style.visibility='visible';\n            kql_MagicLaunchWindowFunction('kqlmagic/temp_files/61221702-0960-441b-ae44-ad08544f3bc9/_f233a343-df06-4d9a-8a18-5b3eb8942c7f_at_loganalytics_schema.html', 'fullscreen=no,directories=no,location=no,menubar=no,resizable=yes,scrollbars=yes,status=no,titlebar=no,toolbar=no,', '_f233a343_df06_4d9a_8a18_5b3eb8942c7f_at_loganalytics_schema', 'https://linda-dev-computer\\n.eastus.instances.azureml.ms\\n');\n            kql_MagicCloseWindow(kql_Magic__f233a343_df06_4d9a_8a18_5b3eb8942c7f_at_loganalytics_schema, -1, -1);\">\n            popup schema f233a343-df06-4d9a-8a18-5b3eb8942c7f@loganalytics</button>\n            </div>\n\n            <script>\n            var kql_Magic__f233a343_df06_4d9a_8a18_5b3eb8942c7f_at_loganalytics_schema = null;\n\n\n            function kql_MagicCloseWindow(window_obj, obj_secs, itself_secs) {\n                if (obj_secs >= 0) {\n                    _timeout = setTimeout(function(){\n                        window_obj.close();\n                        if (itself_secs >= 0) {\n                            __timeout = setTimeout(function(){window.close();}, itself_secs * 1000);\n                        }\n                    }, obj_secs * 1000);\n                } else if (itself_secs >= 0) {\n                    _timeout = setTimeout(function(){window.close();}, itself_secs * 1000);\n                }\n            }\n\n            function kql_MagicLaunchWindowFunction(file_path_or_data, window_params, window_name, host_or_text) {\n                var url;\n                const baseURI = String(window.location);\n                if (host_or_text == 'text' || host_or_text == 'body') {\n                    url = ''\n                } else if (file_path_or_data.startsWith('http')) {\n                    url = file_path_or_data;\n                } else if (host_or_text.endsWith('.azureml.ms') || host_or_text.endsWith('.azureml.net')) {\n                    let azuremlBaseURI = String(window.document.baseURI);\n                    let start = azuremlBaseURI.search('activeFilePath=');\n                    if (start > 0) {\n                        start += 'activeFilePath='.length;\n                        let end = azuremlBaseURI.substring(start).search('&');\n                        if (end < 0) {\n                            end = undefined;\n                        }\n                        let parts = azuremlBaseURI.substring(start, end).split('/');\n                        parts.pop();\n                        url = host_or_text + '/tree/' + parts.join('/') + '/' + file_path_or_data;\n                    } else {\n                        var parts = baseURI.split('/');\n                        parts.pop();\n                        url = parts.join('/') + '/' + file_path_or_data;\n                    }\n                } else {\n                    var base_url = '';\n\n                    // check if azure notebook\n                    var azure_host = (host_or_text == null || host_or_text.length == 0) ? 'https://notebooks.azure.com' : host_or_text;\n                    var start = azure_host.search('//');\n                    var azure_host_suffix = '.' + azure_host.substring(start+2);\n\n                    var end = baseURI.search(azure_host_suffix);\n                    start = baseURI.search('//');\n                    if (start > 0 && end > 0) {\n                        // # azure notebook environment, assume template: https://library-user.notebooks.azure.com\n                        var parts = baseURI.substring(start+2, end).split('-');\n                        if (parts.length == 2) {\n                            var library = parts[0];\n                            var user = parts[1];\n                            base_url = azure_host + '/api/user/' +user+ '/library/' +library+ '/html/';\n                        }\n                    }\n\n                    // check if local jupyter lab\n                    if (base_url.length == 0) {\n                        var configDataScipt  = document.getElementById('jupyter-config-data');\n                        if (configDataScipt != null) {\n                            var jupyterConfigData = JSON.parse(configDataScipt.textContent);\n                            if (jupyterConfigData['appName'] == 'JupyterLab' && jupyterConfigData['serverRoot'] != null &&  jupyterConfigData['treeUrl'] != null) {\n                                var basePath = '/mnt/batch/tasks/shared/LS_root/mounts/clusters/linda-dev-computer/code/Users/linda.hoang' + '/';\n                                if (basePath.startsWith(jupyterConfigData['serverRoot'])) {\n                                    base_url = '/files/' + basePath.substring(jupyterConfigData['serverRoot'].length+1);\n                                }\n                            }\n                        }\n                    }\n\n                    // works for local jupyter notebook\n                    if (base_url.length == 0) {\n\n                        var parts = baseURI.split('/');\n                        parts.pop(); // remove notebook name segment\n                        base_url = parts.join('/') + '/';\n                    }\n                    url = base_url + file_path_or_data;\n                }\n\n                window.focus();\n                var w = screen.width / 2;\n                var h = screen.height / 2;\n                params = 'width='+w+',height='+h;\n                // kql_Magic + window_name should be a global variable \n                window_obj = window.open(url, window_name, window_params + params);\n                if (url == '') {\n                    let decodedData = atob(file_path_or_data);\n                    if (host_or_text == 'text') {\n                        var el = window_obj.document.createElement('p');\n                        window_obj.document.body.overflow = 'auto';\n                        el.style.top = 0;\n                        el.style.left = 0;\n                        el.innerHTML = decodedData;\n                        window_obj.document.body.appendChild(el);\n                    } else {\n                        window_obj.document.body.innerHTML = decodedData;\n                    }\n                }\n                kql_Magic__f233a343_df06_4d9a_8a18_5b3eb8942c7f_at_loganalytics_schema = window_obj;\n            }\n            </script>\n\n            </body></html>"
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "connected\n"
-        }
-      ],
-      "execution_count": 7,
-      "metadata": {
-        "gather": {
-          "logged": 1683619406742
-        },
-        "jupyter": {
-          "outputs_hidden": false,
-          "source_hidden": false
-        },
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Authenticate To Azure Sentinel"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from msticpy.context.azure.sentinel_core import MicrosoftSentinel\n",
-        "\n",
-        "azs = MicrosoftSentinel()\n",
-        "# The Above line SHOULD work without modification. However it often fails to detect local config files when they exist.\n",
-        "# If this happens, uncomments out the below code and replace the variables with your subscription id,\n",
-        "# resource group name, and sentinel workspace name\n",
-        "\n",
-        "#subscription_id = '########-####-####-####-##########'\n",
-        "#resource_group = 'nnnnnnnnn'\n",
-        "#workspace_name = 'nnnnnnnnnnnnnnnnnnnn'\n",
-        "#azs = MicrosoftSentinel(sub_id=subscription_id,res_grp=resource_group, ws_name=workspace_name)\n",
-        "\n",
-        "azs.connect()\n"
-      ],
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": "To sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code FMC5B6R87 to authenticate.\n"
-        }
-      ],
-      "execution_count": 8,
-      "metadata": {
-        "gather": {
-          "logged": 1683619434409
-        }
-      }
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "## Display the results in an interactive browser\n",
-        "\n"
-      ],
-      "metadata": {}
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from ipywidgets import widgets, Layout\n",
-        "from IPython.display import display, HTML, clear_output\n",
-        "from msticpy.nbtools.nbwidgets import QueryTime\n",
-        "\n",
-        "\n",
-        "# Browser Functions\n",
-        "\n",
-        "def on_query_value_change(change):\n",
-        "    if view_qry_check.value:\n",
-        "        qry_text = kql_dict[queries_w.value][1]\n",
-        "        if \"Not convertible\" not in qry_text:\n",
-        "            qry_text = add_filter_clauses(qry_text,\n",
-        "                                          date=add_date_filter_check.value,\n",
-        "                                          host=add_host_filter_check.value)\n",
-        "        query_text_w.value = qry_text.replace('|', '\\n|')\n",
-        "        orig_text_w.value = kql_dict[queries_w.value][0]\n",
-        "        \n",
-        "def clean_kql_comments(query_string):\n",
-        "    \"\"\"Cleans\"\"\"\n",
-        "    import re\n",
-        "    return re.sub(r'(//[^\\n]+)', '', query_string, re.MULTILINE).replace('\\n', '').strip()\n",
-        "\n",
-        "def execute_kql_query(query_string):\n",
-        "    if not query_string or len(query_string.strip()) == 0:\n",
-        "        print('No query supplied')\n",
-        "        return None\n",
-        "    src_query = clean_kql_comments(query_string)\n",
-        "    src_query = src_query.format(start=qry_wgt.start, end=qry_wgt.end)\n",
-        "    result = qry_prov.exec_query(src_query)\n",
-        "    return result\n",
-        "\n",
-        "\n",
-        "def on_view_query_value_change(change):\n",
-        "    vis = 'visible' if view_qry_check.value else 'hidden'\n",
-        "    on_query_value_change(None)\n",
-        "    query_text_w.layout.visibility = vis\n",
-        "    orig_text_w.layout.visibility = vis\n",
-        "    \n",
-        "def display_results(qry_results):\n",
-        "    query_results_w.value = qry_results.to_html()\n",
-        "def parse_severity(string):\n",
-        "    if 'Recorded Future Priority Level' not in string:\n",
-        "        if 'critical' in string:\n",
-        "            return 'High'\n",
-        "        return string.capitalize()\n",
-        "    num = string.split(' ')[-1]\n",
-        "    if num in('0', '1'):\n",
-        "        return 'High'\n",
-        "    elif num == '2':\n",
-        "        return 'Medium'\n",
-        "    elif num == '3':\n",
-        "        return 'Low'\n",
-        "def parse_tactics(tags):\n",
-        "    return [t.split('.', 1)[1] for t in tags]\n",
-        "\n",
-        "def create_analytic_rule(eh):\n",
-        "    query =  list(yaml.safe_load_all(orig_text_w.value))[0]\n",
-        "    \n",
-        "    azs.create_analytic_rule(\n",
-        "        name=query['title'], \n",
-        "        description=f'{query[\"description\"]}. Authored by {query[\"author\"]}. {query[\"date\"]}',\n",
-        "        query=query_text_w.value,\n",
-        "        severity = parse_severity(query['level']),\n",
-        "        #tactics=parse_tactics(query['tags']),\n",
-        "        enabled=False)\n",
-        "    analytic_result.value = f'Created Analytic Rule {query[\"title\"]}'\n",
-        "    \n",
-        "def exec_query(eh):\n",
-        "    query_name = queries_w.value\n",
-        "    query_text = query_text_w.value\n",
-        "    qry_results = execute_kql_query(query_text)\n",
-        "        \n",
-        "    display_results(qry_results)\n",
-        "\n",
-        "\n",
-        "# Browser widget setup\n",
-        "\n",
-        "queries_w = widgets.Select(options = kql_dict.keys(),\n",
-        "                           description='Query :    ',\n",
-        "                           layout=Layout(width='30%', height='120px'),\n",
-        "                           style = {'description_width': 'initial'})\n",
-        "\n",
-        "query_text_w = widgets.Textarea(\n",
-        "    value='',\n",
-        "    description='Kql Query:',\n",
-        "    layout=Layout(width='100%', height='300px', visiblity='hidden'),\n",
-        "    disabled=False)\n",
-        "orig_text_w = widgets.Textarea(\n",
-        "    value='',\n",
-        "    description='Sigma Query:',\n",
-        "    layout=Layout(width='100%', height='250px', visiblity='hidden'),\n",
-        "    disabled=False)\n",
-        "\n",
-        "query_text_w.layout.visibility = 'hidden'\n",
-        "orig_text_w.layout.visibility = 'hidden'\n",
-        "queries_w.observe(on_query_value_change, names='value')\n",
-        "\n",
-        "view_qry_check = widgets.Checkbox(description=\"View query\", value=True)\n",
-        "add_date_filter_check = widgets.Checkbox(description=\"Add date filter\", value=False)\n",
-        "add_host_filter_check = widgets.Checkbox(description=\"Add host filter\", value=False)\n",
-        "\n",
-        "view_qry_check.observe(on_view_query_value_change, names='value')\n",
-        "add_date_filter_check.observe(on_view_query_value_change, names='value')\n",
-        "add_host_filter_check.observe(on_view_query_value_change, names='value')\n",
-        "# view_qry_button.on_click(click_exec_hqry)\n",
-        "# display(exec_hqry_button);\n",
-        "vbox_opts = widgets.VBox([view_qry_check, add_date_filter_check, add_host_filter_check])\n",
-        "hbox = widgets.HBox([queries_w, vbox_opts])\n",
-        "vbox = widgets.VBox([hbox, orig_text_w, query_text_w])\n",
-        "on_view_query_value_change(None)\n",
-        "display(vbox)\n",
-        "query_results_w = widgets.HTML('')\n",
-        "qry_wgt = QueryTime(units='days', before=5, after=0, max_before=30, max_after=10)\n",
-        "display(qry_wgt)\n",
-        "\n",
-        "exec_hqry_button = widgets.Button(description=\"Execute query\")\n",
-        "exec_hqry_button.on_click(exec_query)\n",
-        "\n",
-        "analytic_button = widgets.Button(description = 'Create Analytic Rule')\n",
-        "analytic_button.on_click(create_analytic_rule)\n",
-        "analytic_result = widgets.Label('')\n",
-        "query_vbox = widgets.VBox([exec_hqry_button, query_results_w])\n",
-        "analytic_vbox = widgets.VBox([analytic_button, analytic_result])\n",
-        "final_vbox = widgets.VBox([analytic_vbox, query_vbox])\n",
-        "display(final_vbox)\n"
-      ],
-      "outputs": [
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "VBox(children=(HBox(children=(Select(description='Query :    ', layout=Layout(height='120px', width='30%'), op…",
-            "application/vnd.jupyter.widget-view+json": {
-              "version_major": 2,
-              "version_minor": 0,
-              "model_id": "158d4df56748464bb91d88af118813fa"
-            }
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "VBox(children=(HTML(value='<h4>Set query time boundaries</h4>'), HBox(children=(DatePicker(value=datetime.date…",
-            "application/vnd.jupyter.widget-view+json": {
-              "version_major": 2,
-              "version_minor": 0,
-              "model_id": "e434990f1cfd458faa95378bda9ff780"
-            }
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": "VBox(children=(VBox(children=(Button(description='Create Analytic Rule', style=ButtonStyle()), Label(value='')…",
-            "application/vnd.jupyter.widget-view+json": {
-              "version_major": 2,
-              "version_minor": 0,
-              "model_id": "3e352242a78e4aa08fad1a2216f7f032"
-            }
-          },
-          "metadata": {}
-        },
-        {
-          "output_type": "stream",
-          "name": "stderr",
-          "text": "Error in atexit._run_exitfuncs:\nTraceback (most recent call last):\n  File \"/anaconda/envs/jupyter_env/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3814, in atexit_operations\n    tfile.unlink()\nAttributeError: 'str' object has no attribute 'unlink'\n"
-        }
-      ],
-      "execution_count": 9,
-      "metadata": {
-        "gather": {
-          "logged": 1683619435164
-        },
-        "scrolled": false,
-        "jupyter": {
-          "outputs_hidden": false
-        }
-      }
-    }
-  ],
-  "metadata": {
-    "toc": {
-      "toc_position": {},
-      "skip_h1_title": false,
-      "number_sections": false,
-      "title_cell": "Table of Contents",
-      "toc_window_display": false,
-      "base_numbering": 1,
-      "toc_section_display": true,
-      "title_sidebar": "Contents",
-      "toc_cell": false,
-      "nav_menu": {},
-      "sideBar": true
-    },
-    "hide_input": false,
-    "microsoft": {
-      "ms_spell_check": {
-        "ms_spell_check_language": "en"
-      },
-      "host": {
-        "AzureML": {
-          "notebookHasBeenCompleted": true
-        }
-      }
-    },
-    "kernel_info": {
-      "name": "python38-azureml"
-    },
-    "varInspector": {
-      "cols": {
-        "lenName": 16,
-        "lenType": 16,
-        "lenVar": 40
-      },
-      "kernels_config": {
-        "python": {
-          "delete_cmd_postfix": "",
-          "delete_cmd_prefix": "del ",
-          "library": "var_list.py",
-          "varRefreshCmd": "print(var_dic_list())"
-        },
-        "r": {
-          "delete_cmd_postfix": ") ",
-          "delete_cmd_prefix": "rm(",
-          "library": "var_list.r",
-          "varRefreshCmd": "cat(var_dic_list()) "
-        }
-      },
-      "types_to_exclude": [
-        "module",
-        "function",
-        "builtin_function_or_method",
-        "instance",
-        "_Feature"
-      ],
-      "window_display": false
-    },
-    "language_info": {
-      "name": "python",
-      "version": "3.8.10",
-      "mimetype": "text/x-python",
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "pygments_lexer": "ipython3",
-      "nbconvert_exporter": "python",
-      "file_extension": ".py"
-    },
-    "kernelspec": {
-      "name": "python38-azureml",
-      "language": "python",
-      "display_name": "Python 3.8 - AzureML"
-    },
-    "nteract": {
-      "version": "nteract-front-end@1.0.0"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}
\ No newline at end of file