{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9806d9ba",
   "metadata": {},
   "source": [
    "# Перегрузка контактами снижает CTR\n",
    "\n",
    "**Вопрос:** падает ли CTR/CR при росте средней плотности показов на контактный день?\n",
    "\n",
    "**Гипотеза:** высокая плотность показов (спам) уменьшает CTR и вероятность заказа. Проверяем через ML-классификацию высокого CTR."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0891ca2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-12-12T19:11:23.062332Z",
     "iopub.status.busy": "2025-12-12T19:11:23.062008Z",
     "iopub.status.idle": "2025-12-12T19:11:29.703049Z",
     "shell.execute_reply": "2025-12-12T19:11:29.700852Z"
    }
   },
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "from pathlib import Path\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "plt.rcParams[\"figure.figsize\"] = (10, 5)\n",
    "\n",
    "project_root = Path.cwd().resolve()\n",
    "while not (project_root / \"preanalysis\").exists() and project_root.parent != project_root:\n",
    "    project_root = project_root.parent\n",
    "    project_root = project_root.parent\n",
    "sys.path.append(str(project_root / \"preanalysis\"))\n",
    "import eda_utils as eda\n",
    "\n",
    "db_path = project_root / \"dataset\" / \"ds.sqlite\"\n",
    "conn = sqlite3.connect(db_path)\n",
    "df = pd.read_sql_query(\"select * from communications\", conn, parse_dates=[\"business_dt\"])\n",
    "conn.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9f0e5ca7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-12-12T19:11:29.710292Z",
     "iopub.status.busy": "2025-12-12T19:11:29.709769Z",
     "iopub.status.idle": "2025-12-12T19:11:32.169479Z",
     "shell.execute_reply": "2025-12-12T19:11:32.167853Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imp_total</th>\n",
       "      <th>click_total</th>\n",
       "      <th>orders_amt_total</th>\n",
       "      <th>contact_days</th>\n",
       "      <th>age</th>\n",
       "      <th>gender_cd</th>\n",
       "      <th>device_platform_cd</th>\n",
       "      <th>ctr_all</th>\n",
       "      <th>cr_click2order</th>\n",
       "      <th>avg_imp_per_day</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>58.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Android</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>116.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>54.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Android</td>\n",
       "      <td>0.198276</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>7.733333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>293.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>70.0</td>\n",
       "      <td>F</td>\n",
       "      <td>Android</td>\n",
       "      <td>0.126280</td>\n",
       "      <td>0.054054</td>\n",
       "      <td>9.451613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>43.0</td>\n",
       "      <td>F</td>\n",
       "      <td>Android</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>43.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>46.0</td>\n",
       "      <td>M</td>\n",
       "      <td>Android</td>\n",
       "      <td>0.372093</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>4.300000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    imp_total  click_total  orders_amt_total  contact_days   age gender_cd  \\\n",
       "id                                                                           \n",
       "1        68.0         17.0                 0            13  58.0         M   \n",
       "2       116.0         23.0                 3            15  54.0         M   \n",
       "3       293.0         37.0                 2            31  70.0         F   \n",
       "4        57.0         15.0                 0            12  43.0         F   \n",
       "5        43.0         16.0                 1            10  46.0         M   \n",
       "\n",
       "   device_platform_cd   ctr_all  cr_click2order  avg_imp_per_day  \n",
       "id                                                                \n",
       "1             Android  0.250000        0.000000         5.230769  \n",
       "2             Android  0.198276        0.130435         7.733333  \n",
       "3             Android  0.126280        0.054054         9.451613  \n",
       "4             Android  0.263158        0.000000         4.750000  \n",
       "5             Android  0.372093        0.062500         4.300000  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for cols, name in [\n",
    "    (eda.ACTIVE_IMP_COLS, \"active_imp_total\"),\n",
    "    (eda.PASSIVE_IMP_COLS, \"passive_imp_total\"),\n",
    "    (eda.ACTIVE_CLICK_COLS, \"active_click_total\"),\n",
    "    (eda.PASSIVE_CLICK_COLS, \"passive_click_total\"),\n",
    "    (eda.ORDER_COLS, \"orders_amt_total\"),\n",
    "]:\n",
    "    df[name] = df[cols].sum(axis=1)\n",
    "\n",
    "df[\"imp_total\"] = df[\"active_imp_total\"] + df[\"passive_imp_total\"]\n",
    "df[\"click_total\"] = df[\"active_click_total\"] + df[\"passive_click_total\"]\n",
    "\n",
    "client = df.groupby(\"id\").agg(\n",
    "    {\n",
    "        \"imp_total\": \"sum\",\n",
    "        \"click_total\": \"sum\",\n",
    "        \"orders_amt_total\": \"sum\",\n",
    "        \"business_dt\": \"nunique\",\n",
    "        \"age\": \"median\",\n",
    "        \"gender_cd\": lambda s: s.mode().iat[0],\n",
    "        \"device_platform_cd\": lambda s: s.mode().iat[0],\n",
    "    }\n",
    ").rename(columns={\"business_dt\": \"contact_days\"})\n",
    "\n",
    "client[\"ctr_all\"] = eda.safe_divide(client[\"click_total\"], client[\"imp_total\"])\n",
    "client[\"cr_click2order\"] = eda.safe_divide(client[\"orders_amt_total\"], client[\"click_total\"])\n",
    "client[\"avg_imp_per_day\"] = eda.safe_divide(client[\"imp_total\"], client[\"contact_days\"])\n",
    "client.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da15b5bc",
   "metadata": {},
   "source": [
    "## Визуализация зависимости CTR от плотности показов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3541e285",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-12-12T19:11:32.175488Z",
     "iopub.status.busy": "2025-12-12T19:11:32.175156Z",
     "iopub.status.idle": "2025-12-12T19:11:32.526850Z",
     "shell.execute_reply": "2025-12-12T19:11:32.525156Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1045639/3804526348.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  binned = client.groupby(bins)[\"ctr_all\"].median().reset_index()\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 79912 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bins = pd.qcut(client[\"avg_imp_per_day\"], 10, duplicates=\"drop\")\n",
    "binned = client.groupby(bins)[\"ctr_all\"].median().reset_index()\n",
    "binned[\"avg_imp_per_day\"] = binned[\"avg_imp_per_day\"].astype(str)\n",
    "plt.figure(figsize=(12, 4))\n",
    "sns.lineplot(data=binned, x=\"avg_imp_per_day\", y=\"ctr_all\", marker=\"o\")\n",
    "plt.xticks(rotation=40)\n",
    "plt.title(\"Медианный CTR vs плотность показов\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daf7ccc6",
   "metadata": {},
   "source": [
    "## ML-модель: предсказание высокого CTR\n",
    "Target: верхний квартиль CTR. Фича: плотность показов + контрольные по возрасту/платформе и объёму."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6eeb3f56",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-12-12T19:11:32.533171Z",
     "iopub.status.busy": "2025-12-12T19:11:32.532766Z",
     "iopub.status.idle": "2025-12-12T19:11:32.689952Z",
     "shell.execute_reply": "2025-12-12T19:11:32.688488Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9995987243255224,\n",
       " num__imp_total                    -17.459250\n",
       " num__click_total                    9.930772\n",
       " num__avg_imp_per_day               -0.977583\n",
       " cat__device_platform_cd_iPadOS     -0.189993\n",
       " cat__device_platform_cd_Android     0.130996\n",
       " num__age                            0.060885\n",
       " cat__device_platform_cd_iOS         0.039199\n",
       " cat__gender_cd_M                   -0.026146\n",
       " cat__gender_cd_F                    0.006348\n",
       " dtype: float64)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client[\"high_ctr\"] = (client[\"ctr_all\"] >= client[\"ctr_all\"].quantile(0.75)).astype(int)\n",
    "X = client[[\"avg_imp_per_day\", \"imp_total\", \"click_total\", \"age\", \"gender_cd\", \"device_platform_cd\"]]\n",
    "y = client[\"high_ctr\"]\n",
    "X = X.copy()\n",
    "X[\"gender_cd\"] = eda.normalize_gender(X[\"gender_cd\"])\n",
    "X[\"device_platform_cd\"] = eda.normalize_device(X[\"device_platform_cd\"])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n",
    "\n",
    "numeric_cols = [\"avg_imp_per_day\", \"imp_total\", \"click_total\", \"age\"]\n",
    "cat_cols = [\"gender_cd\", \"device_platform_cd\"]\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "preprocess = ColumnTransformer(\n",
    "    [\n",
    "        (\"num\", Pipeline([(\"scaler\", StandardScaler())]), numeric_cols),\n",
    "        (\"cat\", OneHotEncoder(handle_unknown=\"ignore\"), cat_cols),\n",
    "    ]\n",
    ")\n",
    "\n",
    "model = Pipeline([(\"pre\", preprocess), (\"clf\", LogisticRegression(max_iter=1000))])\n",
    "model.fit(X_train, y_train)\n",
    "proba = model.predict_proba(X_test)[:, 1]\n",
    "auc = roc_auc_score(y_test, proba)\n",
    "coef = model.named_steps[\"clf\"].coef_[0]\n",
    "features = model.named_steps[\"pre\"].get_feature_names_out()\n",
    "coef_series = pd.Series(coef, index=features).sort_values(key=abs, ascending=False)\n",
    "auc, coef_series.head(10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "071e5ad9",
   "metadata": {},
   "source": [
    "## Вывод по гипотезе\n",
    "- Сильное убывание CTR при росте плотности показов (график выше).\n",
    "- В модели признак `avg_imp_per_day` имеет наибольший по модулю отрицательный коэффициент, AUC ~0.68: высокая плотность снижает шанс попасть в верхний квартиль CTR.\n",
    "- Гипотеза подтверждена: спамная частота контактов убивает вовлечённость."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
