{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "bc478b25-756f-4686-8995-a079add279b4", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import json" ] }, { "cell_type": "code", "execution_count": 3, "id": "0a7982a2-a348-43c7-93b8-774605276a60", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
kindapiVersionlevelauditIDstagerequestURIverbsourceIPsuserAgentrequestReceivedTimestamp...annotations_k8s.io/removed-releaseannotations_pod-security.kubernetes.io/enforce-policyannotations_failed-open.mutation.webhook.admission.k8s.io/round_0_index_5annotations_mutation.webhook.admission.k8s.io/round_0_index_5annotations_failed-open.mutation.webhook.admission.k8s.io/round_0_index_7annotations_mutation.webhook.admission.k8s.io/round_0_index_7annotations_mutation.webhook.admission.k8s.io/round_0_index_18annotations_mutation.webhook.admission.k8s.io/round_0_index_16annotationsannotations_mutation.webhook.admission.k8s.io/round_0_index_15
0Eventaudit.k8s.io/v1Metadata4fed3bf5-d1a4-4bf3-a137-bb9ff5380486ResponseStarted/api/v1/nodes?allowWatchBookmarks=true&fieldSe...watch[192.168.60.11]kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc2023-01-27T18:28:08.128513Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1Eventaudit.k8s.io/v1Metadataa1b86d5d-6f8f-4058-9710-19f5cc40a68aResponseStarted/api/v1/namespaces/kube-flannel/configmaps?all...watch[192.168.60.11]kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc2023-01-27T18:28:08.129781Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2Eventaudit.k8s.io/v1Metadataa1b86d5d-6f8f-4058-9710-19f5cc40a68aResponseComplete/api/v1/namespaces/kube-flannel/configmaps?all...watch[192.168.60.11]kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc2023-01-27T18:28:08.129781Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
3Eventaudit.k8s.io/v1Metadata3ba9cc6d-7d10-40c5-8ec7-da60a9b3627cResponseStarted/api/v1/namespaces/kube-system/configmaps?allo...watch[192.168.60.11]kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc2023-01-27T18:28:08.130396Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
4Eventaudit.k8s.io/v1Metadata3ba9cc6d-7d10-40c5-8ec7-da60a9b3627cResponseComplete/api/v1/namespaces/kube-system/configmaps?allo...watch[192.168.60.11]kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc2023-01-27T18:28:08.130396Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
..................................................................
94282Eventaudit.k8s.io/v1Metadataee239268-d276-431e-897b-c9e4ae2ec4a2ResponseComplete/readyzget[192.168.60.10]kube-probe/1.232023-01-27T19:03:15.516370Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
94283Eventaudit.k8s.io/v1Metadatac4027d84-5a11-46f5-808b-cde4c52e68a2ResponseComplete/apis/coordination.k8s.io/v1/namespaces/kube-s...get[192.168.60.10]kube-scheduler/v1.23.15 (linux/amd64) kubernet...2023-01-27T19:03:15.630828Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
94284Eventaudit.k8s.io/v1Metadataccd5ea66-1586-4453-8634-1428d88795f3ResponseComplete/apis/coordination.k8s.io/v1/namespaces/kube-s...get[192.168.60.10]kube-controller-manager/v1.23.15 (linux/amd64)...2023-01-27T19:03:15.644742Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
94285Eventaudit.k8s.io/v1Metadata42728a6e-4b34-406e-89e4-fad679bde27bResponseComplete/apis/coordination.k8s.io/v1/namespaces/kube-s...update[192.168.60.10]kube-scheduler/v1.23.15 (linux/amd64) kubernet...2023-01-27T19:03:15.644221Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
94286Eventaudit.k8s.io/v1Metadatadb1fa802-3c2b-43f3-ad5d-cb09ac400074ResponseComplete/apis/coordination.k8s.io/v1/namespaces/kube-s...update[192.168.60.10]kube-controller-manager/v1.23.15 (linux/amd64)...2023-01-27T19:03:15.653453Z...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

94287 rows × 43 columns

\n", "
" ], "text/plain": [ " kind apiVersion level auditID \\\n", "0 Event audit.k8s.io/v1 Metadata 4fed3bf5-d1a4-4bf3-a137-bb9ff5380486 \n", "1 Event audit.k8s.io/v1 Metadata a1b86d5d-6f8f-4058-9710-19f5cc40a68a \n", "2 Event audit.k8s.io/v1 Metadata a1b86d5d-6f8f-4058-9710-19f5cc40a68a \n", "3 Event audit.k8s.io/v1 Metadata 3ba9cc6d-7d10-40c5-8ec7-da60a9b3627c \n", "4 Event audit.k8s.io/v1 Metadata 3ba9cc6d-7d10-40c5-8ec7-da60a9b3627c \n", "... ... ... ... ... \n", "94282 Event audit.k8s.io/v1 Metadata ee239268-d276-431e-897b-c9e4ae2ec4a2 \n", "94283 Event audit.k8s.io/v1 Metadata c4027d84-5a11-46f5-808b-cde4c52e68a2 \n", "94284 Event audit.k8s.io/v1 Metadata ccd5ea66-1586-4453-8634-1428d88795f3 \n", "94285 Event audit.k8s.io/v1 Metadata 42728a6e-4b34-406e-89e4-fad679bde27b \n", "94286 Event audit.k8s.io/v1 Metadata db1fa802-3c2b-43f3-ad5d-cb09ac400074 \n", "\n", " stage requestURI \\\n", "0 ResponseStarted /api/v1/nodes?allowWatchBookmarks=true&fieldSe... \n", "1 ResponseStarted /api/v1/namespaces/kube-flannel/configmaps?all... \n", "2 ResponseComplete /api/v1/namespaces/kube-flannel/configmaps?all... \n", "3 ResponseStarted /api/v1/namespaces/kube-system/configmaps?allo... \n", "4 ResponseComplete /api/v1/namespaces/kube-system/configmaps?allo... \n", "... ... ... \n", "94282 ResponseComplete /readyz \n", "94283 ResponseComplete /apis/coordination.k8s.io/v1/namespaces/kube-s... \n", "94284 ResponseComplete /apis/coordination.k8s.io/v1/namespaces/kube-s... \n", "94285 ResponseComplete /apis/coordination.k8s.io/v1/namespaces/kube-s... \n", "94286 ResponseComplete /apis/coordination.k8s.io/v1/namespaces/kube-s... \n", "\n", " verb sourceIPs \\\n", "0 watch [192.168.60.11] \n", "1 watch [192.168.60.11] \n", "2 watch [192.168.60.11] \n", "3 watch [192.168.60.11] \n", "4 watch [192.168.60.11] \n", "... ... ... \n", "94282 get [192.168.60.10] \n", "94283 get [192.168.60.10] \n", "94284 get [192.168.60.10] \n", "94285 update [192.168.60.10] \n", "94286 update [192.168.60.10] \n", "\n", " userAgent \\\n", "0 kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc \n", "1 kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc \n", "2 kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc \n", "3 kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc \n", "4 kubelet/v1.23.14 (linux/amd64) kubernetes/3321ffc \n", "... ... \n", "94282 kube-probe/1.23 \n", "94283 kube-scheduler/v1.23.15 (linux/amd64) kubernet... \n", "94284 kube-controller-manager/v1.23.15 (linux/amd64)... \n", "94285 kube-scheduler/v1.23.15 (linux/amd64) kubernet... \n", "94286 kube-controller-manager/v1.23.15 (linux/amd64)... \n", "\n", " requestReceivedTimestamp ... annotations_k8s.io/removed-release \\\n", "0 2023-01-27T18:28:08.128513Z ... NaN \n", "1 2023-01-27T18:28:08.129781Z ... NaN \n", "2 2023-01-27T18:28:08.129781Z ... NaN \n", "3 2023-01-27T18:28:08.130396Z ... NaN \n", "4 2023-01-27T18:28:08.130396Z ... NaN \n", "... ... ... ... \n", "94282 2023-01-27T19:03:15.516370Z ... NaN \n", "94283 2023-01-27T19:03:15.630828Z ... NaN \n", "94284 2023-01-27T19:03:15.644742Z ... NaN \n", "94285 2023-01-27T19:03:15.644221Z ... NaN \n", "94286 2023-01-27T19:03:15.653453Z ... NaN \n", "\n", " annotations_pod-security.kubernetes.io/enforce-policy \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_failed-open.mutation.webhook.admission.k8s.io/round_0_index_5 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_mutation.webhook.admission.k8s.io/round_0_index_5 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_failed-open.mutation.webhook.admission.k8s.io/round_0_index_7 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_mutation.webhook.admission.k8s.io/round_0_index_7 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_mutation.webhook.admission.k8s.io/round_0_index_18 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_mutation.webhook.admission.k8s.io/round_0_index_16 \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations \\\n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", " annotations_mutation.webhook.admission.k8s.io/round_0_index_15 \n", "0 NaN \n", "1 NaN \n", "2 NaN \n", "3 NaN \n", "4 NaN \n", "... ... \n", "94282 NaN \n", "94283 NaN \n", "94284 NaN \n", "94285 NaN \n", "94286 NaN \n", "\n", "[94287 rows x 43 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "file_path = \"../../raw-audit-logs.log\"\n", "\n", "# Set the chunk size (number of rows to process at a time)\n", "chunk_size = 1000 # Adjust based on your available memory\n", "\n", "# Create an empty list to hold the chunks\n", "chunks = []\n", "\n", "# Iterate over the file in chunks\n", "try:\n", " for chunk in pd.read_json(file_path, lines=True, chunksize=chunk_size, encoding_errors='ignore'):\n", " # Append each chunk to the list\n", " chunks.append(chunk)\n", "except ValueError as e:\n", " print(f\"Error while parsing JSON: {e}\")\n", "\n", "# Combine all chunks into a single DataFrame\n", "df = pd.concat(chunks, ignore_index=True)\n", "\n", "# Display the first few rows of the DataFrame\n", "#print(df.head())\n", "#df['user_parsed'] = df['user'].apply(json.loads)\n", "#df_exploded = df.explode('user')\n", "\n", "df_normalized = pd.json_normalize(df.to_dict(orient='records'),\n", "# record_path=['details'],\n", "# meta=['timestamp', 'event', ['user', 'id'], ['user', 'name'], ['user', 'email']],\n", " sep='_'\n", ")\n", "\n", "#df.head()\n", "#df_normalized.info()\n", "df_normalized.iloc[0:2]\n", "df_normalized" ] }, { "cell_type": "code", "execution_count": 224, "id": "c11f7c76-4453-4829-879e-7ac7f1faf541", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2163" ] }, "execution_count": 224, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "auditLogDF = pd.read_csv(r\"data/audit-logs.csv\", names=['hostIP', 'Timestamp', 'username', 'verb', 'resourceType', 'subresource', 'ObjectName', 'ObjectName1', 'requestURI', 'auditID', 'stage', 'statusCode', 'UID', 'ReferenceUID', 'UID1'])\n", "\n", "#print(auditLogDF)\n", "\n", "#print(auditLogDF.count)\n", "auditLogDF = auditLogDF.drop_duplicates(ignore_index=True)\n", "\n", "auditLogDF = auditLogDF.drop(auditLogDF.columns[[0, 2, 5, 8, 9, 10, 11]], axis=1)\n", "auditLogDF = auditLogDF[\n", " (auditLogDF['verb'] != \"get\") &\n", " (auditLogDF['verb'] != \"watch\") &\n", " (auditLogDF['verb'] != \"list\") &\n", " (auditLogDF['resourceType'] != \"events\") &\n", " (auditLogDF['resourceType'] != \"leases\")\n", " ]\n", "\n", "#print(auditLogDF.count)\n", "auditLogDF = auditLogDF[auditLogDF.resourceType != \"replicationcontrollers\"]\n", "\n", "auditLogDF[\"event_type\"] = auditLogDF[\"verb\"] + \"_\" + auditLogDF[\"resourceType\"]\n", "#auditLogDF = auditLogDF[auditLogDF.event_type != \"delete_service\"]\n", "#auditLogDF = auditLogDF[auditLogDF.event_type != \"create namespaces\"]\n", "auditLogDF['ObjectName'] = auditLogDF['ObjectName'].combine_first(auditLogDF['ObjectName1'])\n", "auditLogDF[\"UID\"] = auditLogDF[\"UID\"].fillna(auditLogDF['UID1'])\n", "auditLogDF = auditLogDF.drop(auditLogDF.columns[[4, 7]], axis=1)\n", "auditLogDF[\"UID\"] = auditLogDF[\"ReferenceUID\"].fillna(auditLogDF['UID'])\n", "#auditLogDF = auditLogDF.sort_values([\"UID\", \"Timestamp\"], ascending = (False, True))\n", "auditLogDF = auditLogDF.drop(auditLogDF.columns[[0, 5]], axis=1)\n", "#auditLogDF = auditLogDF.drop_duplicates()\n", "\n", "#print(auditLogDF.to_string())\n", "#print(auditLogDF.count)\n", "#print(auditLogDF)\n", "\n", "#len(auditLogDF)" ] }, { "cell_type": "code", "execution_count": 227, "id": "6dce47db-06fb-4dca-8eda-056118f7459b", "metadata": {}, "outputs": [], "source": [ "\n", "event_seq = []\n", "event_sub_seq = []\n", "for c, r in auditLogDF.iterrows():\n", " with open(\"data/events-dataset-audit.txt\", \"a\") as event_file:\n", " event_file.write(\"%s\\n\" % r['event_type'])" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.2" } }, "nbformat": 4, "nbformat_minor": 5 }