{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# KeyChecker\n",
        "\n",
        "This presentations goal it to introduce the features of the `KeyChecker` and how to configure it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### The challenge\n",
        "\n",
        "I want to ensure, that the log format does not change unnoticed.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "given log entry:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "document = {\n",
        "    \"_systemd_owner_uid\": \"1000\",\n",
        "    \"_systemd_session\": \"198\",\n",
        "    \"_hostname\": \"dev-machine\",\n",
        "    \"_audit_session\": \"198\",\n",
        "    \"_boot_id\": \"3eef443102284373bb33022da6c23d2b\",\n",
        "    \"_systemd_unit\": \"session-198.scope\",\n",
        "    \"_transport\": \"syslog\",\n",
        "    \"_pid\": \"712694\",\n",
        "    \"_cmdline\": \"/usr/bin/sudo journalctl --no-pager -o json -f\",\n",
        "    \"_cap_effective\": \"1ffffffffff\",\n",
        "    \"__monotonic_timestamp\": \"263250766668\",\n",
        "    \"_selinux_context\": \"unconfined\\n\",\n",
        "    \"__realtime_timestamp\": \"1667914601897529\",\n",
        "    \"_gid\": \"0\",\n",
        "    \"_uid\": \"1000\",\n",
        "    \"_systemd_cgroup\": \"/user.slice/user-1000.slice/session-198.scope\",\n",
        "    \"_comm\": \"sudo\",\n",
        "    \"_audit_loginuid\": \"1000\",\n",
        "    \"_systemd_slice\": \"user-1000.slice\",\n",
        "    \"_machine_id\": \"edafb3b3b3ed4d7a8878309023f456fe\",\n",
        "    \"syslog_timestamp\": \"nov  8 13:36:41 \",\n",
        "    \"message\": \"pam_unix(sudo:session): session opened for user root(uid=0) by vagrant(uid=1000)\",\n",
        "    \"_systemd_user_slice\": \"-.slice\",\n",
        "    \"syslog_identifier\": \"sudo\",\n",
        "    \"_systemd_invocation_id\": \"19bb831be8c04629b4df55edf5b3bdcb\",\n",
        "    \"syslog_facility\": \"10\",\n",
        "    \"__cursor\": \"s=99e63e2c458b47fcbad587fb0e74be0d;i=21fd41;b=3eef443102284373bb33022da6c23d2b;m=3d4af7eb4c;t=5ecf5a15c9e39;x=36322c30d547bfb8\",\n",
        "    \"priority\": \"6\",\n",
        "    \"_exe\": \"/usr/bin/sudo\",\n",
        "    \"_source_realtime_timestamp\": \"1667914601879236\",\n",
        "}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "to this:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "expected = {\n",
        "    \"_systemd_owner_uid\": \"1000\",\n",
        "    \"_systemd_session\": \"198\",\n",
        "    \"_hostname\": \"dev-machine\",\n",
        "    \"_audit_session\": \"198\",\n",
        "    \"_boot_id\": \"3eef443102284373bb33022da6c23d2b\",\n",
        "    \"_systemd_unit\": \"session-198.scope\",\n",
        "    \"_transport\": \"syslog\",\n",
        "    \"_pid\": \"712694\",\n",
        "    \"_cmdline\": \"/usr/bin/sudo journalctl --no-pager -o json -f\",\n",
        "    \"_cap_effective\": \"1ffffffffff\",\n",
        "    \"__monotonic_timestamp\": \"263250766668\",\n",
        "    \"_selinux_context\": \"unconfined\\n\",\n",
        "    \"__realtime_timestamp\": \"1667914601897529\",\n",
        "    \"_gid\": \"0\",\n",
        "    \"_uid\": \"1000\",\n",
        "    \"_systemd_cgroup\": \"/user.slice/user-1000.slice/session-198.scope\",\n",
        "    \"_comm\": \"sudo\",\n",
        "    \"_audit_loginuid\": \"1000\",\n",
        "    \"_systemd_slice\": \"user-1000.slice\",\n",
        "    \"_machine_id\": \"edafb3b3b3ed4d7a8878309023f456fe\",\n",
        "    \"syslog_timestamp\": \"nov  8 13:36:41 \",\n",
        "    \"message\": \"pam_unix(sudo:session): session opened for user root(uid=0) by vagrant(uid=1000)\",\n",
        "    \"_systemd_user_slice\": \"-.slice\",\n",
        "    \"syslog_identifier\": \"sudo\",\n",
        "    \"_systemd_invocation_id\": \"19bb831be8c04629b4df55edf5b3bdcb\",\n",
        "    \"syslog_facility\": \"10\",\n",
        "    \"__cursor\": \"s=99e63e2c458b47fcbad587fb0e74be0d;i=21fd41;b=3eef443102284373bb33022da6c23d2b;m=3d4af7eb4c;t=5ecf5a15c9e39;x=36322c30d547bfb8\",\n",
        "    \"priority\": \"6\",\n",
        "    \"_exe\": \"/usr/bin/sudo\",\n",
        "    \"_source_realtime_timestamp\": \"1667914601879236\",\n",
        "}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Create rule and processor"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the rule:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "675"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from pathlib import Path\n",
        "import sys\n",
        "sys.path.append(\"../../../../../\")\n",
        "import tempfile\n",
        "\n",
        "\n",
        "rule_yaml = \"\"\"---\n",
        "filter: \"message\"\n",
        "key_checker:\n",
        "  source_fields:\n",
        "    - _systemd_owner_uid\n",
        "    - _systemd_session\n",
        "    - _hostname \n",
        "    - _audit_session\n",
        "    - _boot_id\n",
        "    - _systemd_unit\n",
        "    - _transport\n",
        "    - _pid\n",
        "    - _cmdline\n",
        "    - _cap_effective\n",
        "    - __monotonic_timestamp\n",
        "    - _selinux_context\n",
        "    - __realtime_timestamp\n",
        "    - _gid\n",
        "    - _uid\n",
        "    - _systemd_cgroup\n",
        "    - _comm\n",
        "    - _audit_loginuid\n",
        "    - _systemd_slice\n",
        "    - _machine_id\n",
        "    - syslog_timestamp\n",
        "    - message\n",
        "    - _systemd_user_slice\n",
        "    - syslog_identifier\n",
        "    - _systemd_invocation_id\n",
        "    - syslog_facility\n",
        "    - __cursor\n",
        "    - priority\n",
        "    - _exe\n",
        "    - _source_realtime_timestamp\n",
        "  target_field: missing_fields\n",
        "\"\"\"\n",
        "\n",
        "rule_path = Path(tempfile.gettempdir()) / \"concatenator\"\n",
        "rule_path.mkdir(exist_ok=True)\n",
        "rule_file = rule_path / \"data-stream.yml\"\n",
        "rule_file.write_text(rule_yaml)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor config:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "processor_config = {\n",
        "    \"almighty_keychecker\": {\n",
        "        \"type\": \"key_checker\",\n",
        "        \"rules\": [str(rule_path), \"/dev\"],\n",
        "    }\n",
        "}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor with the factory:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "ename": "ModuleNotFoundError",
          "evalue": "No module named 'logprep'",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39msys\u001b[39;00m\n\u001b[1;32m      3\u001b[0m sys\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mappend(\u001b[39m\"\u001b[39m\u001b[39m..\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mlogprep\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mfactory\u001b[39;00m \u001b[39mimport\u001b[39;00m Factory\n\u001b[1;32m      6\u001b[0m mock_logger \u001b[39m=\u001b[39m mock\u001b[39m.\u001b[39mMagicMock()\n\u001b[1;32m      7\u001b[0m keychecker \u001b[39m=\u001b[39m Factory\u001b[39m.\u001b[39mcreate(processor_config, mock_logger)\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'logprep'"
          ]
        }
      ],
      "source": [
        "from unittest import mock\n",
        "import sys\n",
        "from logprep.factory import Factory\n",
        "\n",
        "mock_logger = mock.MagicMock()\n",
        "keychecker = Factory.create(processor_config)\n",
        "keychecker\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Process event\n",
        "\n",
        "#### Case 1 - nothing changed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from copy import deepcopy\n",
        "\n",
        "mydocument = deepcopy(document)\n",
        "keychecker.process(mydocument)\n",
        "assert mydocument == expected\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Case 2 - missing field"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "False\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['_hostname', 'syslog_timestamp']"
            ]
          },
          "execution_count": 58,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "mydocument.pop(\"_hostname\")\n",
        "mydocument.pop(\"syslog_timestamp\")\n",
        "keychecker.process(mydocument)\n",
        "print(mydocument == expected)\n",
        "mydocument[\"missing_fields\"]"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.10.6 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}