{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Concatenator\n",
        "\n",
        "This presentations goal it to introduce the features of the `Concatenator` and how to configure it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### The challenge\n",
        "\n",
        "I want to merge different fields from an event in one target field."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "from this:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {},
      "outputs": [],
      "source": [
        "document = {\n",
        "    'data_stream': {\n",
        "        'dataset': 'windows', \n",
        "        'namespace': 'devopslab', \n",
        "        'type': 'logs'\n",
        "        }, \n",
        "    '_op_type': 'create'\n",
        "    }"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "to this:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {},
      "outputs": [],
      "source": [
        "expected = {\n",
        "    'data_stream': {\n",
        "        'dataset': 'windows', \n",
        "        'namespace': 'devopslab', \n",
        "        'type': 'logs'\n",
        "        }, \n",
        "    '_op_type': 'create', \n",
        "    '_index': 'logs-windows-devopslab'\n",
        "    }"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Create rule and processor"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the rule:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "230"
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import sys\n",
        "sys.path.append(\"../../../../../\")\n",
        "import tempfile\n",
        "from pathlib import Path\n",
        "\n",
        "rule_yaml = \"\"\"---\n",
        "filter: \"data_stream\"\n",
        "concatenator:\n",
        "  source_fields:\n",
        "    - data_stream.type\n",
        "    - data_stream.dataset\n",
        "    - data_stream.namespace\n",
        "  target_field: _index\n",
        "  separator: \"-\"\n",
        "  overwrite_target: false\n",
        "  delete_source_fields: false\n",
        "\"\"\"\n",
        "\n",
        "rule_path = Path(tempfile.gettempdir()) / \"concatenator\"\n",
        "rule_path.mkdir(exist_ok=True)\n",
        "rule_file = rule_path / \"data-stream.yml\"\n",
        "rule_file.write_text(rule_yaml)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor config:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {},
      "outputs": [],
      "source": [
        "processor_config = {\n",
        "    \"myconcatenator\":{   \n",
        "        \"type\": \"concatenator\",\n",
        "        \"rules\": [str(rule_path), \"/dev\"],\n",
        "        }\n",
        "    }"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "create the processor with the factory:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "concatenator"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from unittest import mock\n",
        "from logprep.factory import Factory\n",
        "\n",
        "mock_logger = mock.MagicMock()\n",
        "concatenator = Factory.create(processor_config)\n",
        "concatenator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Process event"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "before: {'data_stream': {'dataset': 'windows', 'namespace': 'devopslab', 'type': 'logs'}, '_op_type': 'create'}\n",
            "after: {'data_stream': {'dataset': 'windows', 'namespace': 'devopslab', 'type': 'logs'}, '_op_type': 'create', '_index': 'logs-windows-devopslab'}\n",
            "True\n"
          ]
        }
      ],
      "source": [
        "from copy import deepcopy\n",
        "mydocument = deepcopy(document)\n",
        "\n",
        "\n",
        "print(f\"before: {mydocument}\")\n",
        "concatenator.process(mydocument)\n",
        "print(f\"after: {mydocument}\")\n",
        "print(mydocument == expected)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.11.0 ('.venv': venv)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.0"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "586280540a85d3e21edc698fe7b86af2848b9b02644e6c22463da25c40a3f1be"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}