{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In Short"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/valery/Documents/_code/arche/src\n"
     ]
    }
   ],
   "source": [
    "%cd ../../../src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from arche import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema = {\n",
    "    \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n",
    "    \"definitions\": {\n",
    "        \"float\": {\n",
    "            \"pattern\": \"^-?[0-9]+\\\\.[0-9]{2}$\"\n",
    "        },\n",
    "        \"url\": {\n",
    "            \"pattern\": \"^https?://(www\\\\.)?[a-z0-9.-]*\\\\.[a-z]{2,}([^<>%\\\\x20\\\\x00-\\\\x1f\\\\x7F]|%[0-9a-fA-F]{2})*$\"\n",
    "        }\n",
    "    },\n",
    "    \"additionalProperties\": False,\n",
    "    \"type\": \"object\",\n",
    "    \"properties\": {\n",
    "        \"category\": {\"type\": \"string\", \"tag\": [\"category\"]},\n",
    "        \"price\": {\"type\": \"string\", \"pattern\": \"^£\\d{2}.\\d{2}$\"},\n",
    "        \"_type\": {\"type\": \"string\"},\n",
    "        \"description\": {\"type\": \"string\"},\n",
    "        \"title\": {\"type\": \"string\"},\n",
    "        \"_key\": {\"type\": \"string\"}\n",
    "    },\n",
    "    \"required\": [\n",
    "        \"_key\",\n",
    "        \"_type\",\n",
    "        \"category\",\n",
    "        \"description\",\n",
    "        \"price\",\n",
    "        \"title\"\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Arche(\"381798/1/2\", schema=schema, target=\"381798/1/1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9613f9e4ae3344fcad48debef2ddb949",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Fetching 0:1000 from 381798/1/2', max=1000, style=ProgressSty…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_key</th>\n",
       "      <th>_type</th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://app.scrapinghub.com/p/381798/1/2/item/0</td>\n",
       "      <td>dict</td>\n",
       "      <td>Young Adult</td>\n",
       "      <td>Patient Twenty-nine.A monster roams the halls ...</td>\n",
       "      <td>£22.65</td>\n",
       "      <td>The Requiem Red</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://app.scrapinghub.com/p/381798/1/2/item/1</td>\n",
       "      <td>dict</td>\n",
       "      <td>History</td>\n",
       "      <td>From a renowned historian comes a groundbreaki...</td>\n",
       "      <td>£54.23</td>\n",
       "      <td>Sapiens: A Brief History of Humankind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://app.scrapinghub.com/p/381798/1/2/item/2</td>\n",
       "      <td>dict</td>\n",
       "      <td>Mystery</td>\n",
       "      <td>WICKED above her hipbone, GIRL across her hear...</td>\n",
       "      <td>£47.82</td>\n",
       "      <td>Sharp Objects</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://app.scrapinghub.com/p/381798/1/2/item/3</td>\n",
       "      <td>dict</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Dans une France assez proche de la nôtre, un h...</td>\n",
       "      <td>£50.10</td>\n",
       "      <td>Soumission</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://app.scrapinghub.com/p/381798/1/2/item/4</td>\n",
       "      <td>dict</td>\n",
       "      <td>Historical Fiction</td>\n",
       "      <td>\"Erotic and absorbing...Written with starling ...</td>\n",
       "      <td>£53.74</td>\n",
       "      <td>Tipping the Velvet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              _key _type            category  \\\n",
       "0  https://app.scrapinghub.com/p/381798/1/2/item/0  dict         Young Adult   \n",
       "1  https://app.scrapinghub.com/p/381798/1/2/item/1  dict             History   \n",
       "2  https://app.scrapinghub.com/p/381798/1/2/item/2  dict             Mystery   \n",
       "3  https://app.scrapinghub.com/p/381798/1/2/item/3  dict             Fiction   \n",
       "4  https://app.scrapinghub.com/p/381798/1/2/item/4  dict  Historical Fiction   \n",
       "\n",
       "                                         description   price  \\\n",
       "0  Patient Twenty-nine.A monster roams the halls ...  £22.65   \n",
       "1  From a renowned historian comes a groundbreaki...  £54.23   \n",
       "2  WICKED above her hipbone, GIRL across her hear...  £47.82   \n",
       "3  Dans une France assez proche de la nôtre, un h...  £50.10   \n",
       "4  \"Erotic and absorbing...Written with starling ...  £53.74   \n",
       "\n",
       "                                   title  \n",
       "0                        The Requiem Red  \n",
       "1  Sapiens: A Brief History of Humankind  \n",
       "2                          Sharp Objects  \n",
       "3                             Soumission  \n",
       "4                     Tipping the Velvet  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.source_items.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "401cc704bddc45e58565ed09e9375784",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='JSON Schema Validation', max=1000, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Job Outcome:\n",
      "\tFinished\n",
      "\n",
      "Job Errors:\n",
      "\tNo errors\n",
      "\n",
      "Responses Per Item Ratio:\n",
      "\tNumber of responses / Number of scraped items - 1.05\n",
      "\n",
      "Total Scraped Items:\n",
      "\tSame number of items\n",
      "\n",
      "Compare Runtime:\n",
      "\tSimilar or better runtime - 0:00:49.589000 and 0:00:55.089000\n",
      "\n",
      "Finish Time:\n",
      "\tLess than 1 day difference\n",
      "\n",
      "Fields Coverage:\n",
      "\tPASSED\n",
      "\n",
      "Boolean Fields:\n",
      "\tSKIPPED\n",
      "\n",
      "JSON Schema Validation:\n",
      "\u001b[31m\u001b[0m\t1000 items were checked, 1 error(s)\u001b[0m\n",
      "\n",
      "Tags:\n",
      "\tUsed - category\n",
      "\tNot used - name_field, product_price_field, product_price_was_field, product_url_field, unique\n",
      "\n",
      "Compare Price Was And Now:\n",
      "\tproduct_price_field or product_price_was_field tags were not found in schema\n",
      "\n",
      "Uniqueness:\n",
      "\t'unique' tag was not found in schema\n",
      "\n",
      "Duplicated Items:\n",
      "\t'name_field' and 'product_url_field' tags were not found in schema\n",
      "\n",
      "Coverage For Scraped Categories:\n",
      "\t50 categories in 'category'\n",
      "\n",
      "Compare Prices For Same Urls:\n",
      "\tproduct_url_field tag is not set\n",
      "\n",
      "Compare Names Per Url:\n",
      "\tproduct_url_field tag is not set\n",
      "\n",
      "Compare Prices For Same Names:\n",
      "\tname_field tag is not set\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coverage Difference (1 message(s)):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><div id=\"bf6a996e-c3df-4215-bac4-74acad1ab686\" style=\"height: 450px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "if (document.getElementById(\"bf6a996e-c3df-4215-bac4-74acad1ab686\")) {\n",
       "    Plotly.newPlot(\"bf6a996e-c3df-4215-bac4-74acad1ab686\", [{\"name\": \"381798/1/2\", \"orientation\": \"h\", \"x\": [1.0, 1.0, 1.0, 1.0, 1.0], \"y\": [\"_type\", \"category\", \"description\", \"price\", \"title\"], \"type\": \"bar\", \"uid\": \"12b87807-f0b2-4be8-90f1-6674f44236fe\"}, {\"name\": \"381798/1/1\", \"orientation\": \"h\", \"x\": [1.0, 1.0, 1.0, 1.0, 1.0], \"y\": [\"_type\", \"category\", \"description\", \"price\", \"title\"], \"type\": \"bar\", \"uid\": \"10239d21-2cca-4609-9cc2-f6f690acd919\"}], {\"bargap\": 0.1, \"height\": 450, \"hovermode\": \"y\", \"margin\": {\"l\": 200, \"t\": 35}, \"template\": {\"data\": {\"barpolar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"bar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"bar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"baxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"choropleth\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contourcarpet\"}], \"contour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contour\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmapgl\"}], \"heatmap\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmap\"}], \"histogram2dcontour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2dcontour\"}], \"histogram2d\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2d\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"histogram\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"parcoords\"}], \"scatter3d\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattermapbox\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolargl\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolar\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"rgb(237,237,237)\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"rgb(217,217,217)\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"sequential\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]], \"sequentialminus\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]]}, \"colorway\": [\"#F8766D\", \"#A3A500\", \"#00BF7D\", \"#00B0F6\", \"#E76BF3\"], \"font\": {\"color\": \"rgb(51,51,51)\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"rgb(237,237,237)\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hovermode\": \"closest\", \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"rgb(237,237,237)\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"fillcolor\": \"black\", \"line\": {\"width\": 0}, \"opacity\": 0.3}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}}, \"title\": {\"text\": \"Coverage from job stats fields counts\"}, \"xaxis\": {\"range\": [0, 1.05], \"tickformat\": \".2p\"}}, {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\"}); \n",
       "}\n",
       "</script><script type=\"text/javascript\">window.addEventListener(\"resize\", function(){if (document.getElementById(\"bf6a996e-c3df-4215-bac4-74acad1ab686\")) {Plotly.Plots.resize(document.getElementById(\"bf6a996e-c3df-4215-bac4-74acad1ab686\"));};})</script></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fields Coverage (1 message(s)):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><div id=\"0e86c4d4-19d2-44d6-8229-3fc7b6dbdf57\" style=\"height: 450px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "if (document.getElementById(\"0e86c4d4-19d2-44d6-8229-3fc7b6dbdf57\")) {\n",
       "    Plotly.newPlot(\"0e86c4d4-19d2-44d6-8229-3fc7b6dbdf57\", [{\"orientation\": \"h\", \"x\": [1000, 1000, 1000, 998], \"y\": [\"title\", \"price\", \"category\", \"description\"], \"type\": \"bar\", \"uid\": \"308f1d7f-ba61-49da-8b2d-b19bc4512be0\"}], {\"annotations\": [{\"showarrow\": false, \"text\": \"99.80%\", \"x\": 0, \"xref\": \"paper\", \"y\": \"description\", \"yref\": \"y\"}, {\"showarrow\": false, \"text\": \"100.00%\", \"x\": 0, \"xref\": \"paper\", \"y\": \"category\", \"yref\": \"y\"}], \"bargap\": 0.1, \"height\": 450, \"hovermode\": \"y\", \"margin\": {\"l\": 200, \"t\": 35}, \"template\": {\"data\": {\"barpolar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"bar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"bar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"baxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"choropleth\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contourcarpet\"}], \"contour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contour\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmapgl\"}], \"heatmap\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmap\"}], \"histogram2dcontour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2dcontour\"}], \"histogram2d\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2d\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"histogram\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"parcoords\"}], \"scatter3d\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattermapbox\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolargl\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolar\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"rgb(237,237,237)\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"rgb(217,217,217)\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"sequential\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]], \"sequentialminus\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]]}, \"colorway\": [\"#F8766D\", \"#A3A500\", \"#00BF7D\", \"#00B0F6\", \"#E76BF3\"], \"font\": {\"color\": \"rgb(51,51,51)\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"rgb(237,237,237)\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hovermode\": \"closest\", \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"rgb(237,237,237)\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"fillcolor\": \"black\", \"line\": {\"width\": 0}, \"opacity\": 0.3}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}}, \"title\": {\"text\": \"Fields coverage for 1_000 items\"}, \"xaxis\": {\"range\": [0, 1050.0]}}, {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\"}); \n",
       "}\n",
       "</script><script type=\"text/javascript\">window.addEventListener(\"resize\", function(){if (document.getElementById(\"0e86c4d4-19d2-44d6-8229-3fc7b6dbdf57\")) {Plotly.Plots.resize(document.getElementById(\"0e86c4d4-19d2-44d6-8229-3fc7b6dbdf57\"));};})</script></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "JSON Schema Validation (1 message(s)):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "2 items affected - description is not of type 'string': <a href='https://app.scrapinghub.com/p/381798/1/2/item/459'>459</a> <a href='https://app.scrapinghub.com/p/381798/1/2/item/982'>982</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coverage For Scraped Categories (1 message(s)):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><div id=\"54cef0ef-2376-4d77-98bb-53d63d42eea6\" style=\"height: 900px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "if (document.getElementById(\"54cef0ef-2376-4d77-98bb-53d63d42eea6\")) {\n",
       "    Plotly.newPlot(\"54cef0ef-2376-4d77-98bb-53d63d42eea6\", [{\"orientation\": \"h\", \"x\": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 7, 7, 8, 9, 10, 11, 11, 11, 12, 13, 14, 16, 17, 17, 18, 19, 19, 26, 29, 30, 32, 35, 48, 54, 65, 67, 75, 110, 152], \"y\": [\"Short Stories\", \"Adult Fiction\", \"Cultural\", \"Crime\", \"Parenting\", \"Academic\", \"Paranormal\", \"Erotica\", \"Suspense\", \"Novels\", \"Historical\", \"Contemporary\", \"Politics\", \"Christian\", \"Health\", \"Self Help\", \"Biography\", \"Sports and Games\", \"Spirituality\", \"Christian Fiction\", \"New Adult\", \"Psychology\", \"Religion\", \"Art\", \"Autobiography\", \"Humor\", \"Travel\", \"Philosophy\", \"Thriller\", \"Business\", \"Music\", \"Science\", \"Science Fiction\", \"Horror\", \"Womens Fiction\", \"History\", \"Classics\", \"Poetry\", \"Historical Fiction\", \"Childrens\", \"Food and Drink\", \"Mystery\", \"Romance\", \"Fantasy\", \"Young Adult\", \"Fiction\", \"Add a comment\", \"Sequential Art\", \"Nonfiction\", \"Default\"], \"type\": \"bar\", \"uid\": \"07fa5542-e039-447e-a04f-e59af7210812\"}], {\"bargap\": 0.1, \"height\": 900, \"hovermode\": \"y\", \"margin\": {\"l\": 200, \"t\": 35}, \"template\": {\"data\": {\"barpolar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"bar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"bar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"baxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"choropleth\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contourcarpet\"}], \"contour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contour\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmapgl\"}], \"heatmap\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmap\"}], \"histogram2dcontour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2dcontour\"}], \"histogram2d\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2d\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"histogram\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"parcoords\"}], \"scatter3d\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattermapbox\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolargl\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolar\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"rgb(237,237,237)\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"rgb(217,217,217)\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"sequential\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]], \"sequentialminus\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]]}, \"colorway\": [\"#F8766D\", \"#A3A500\", \"#00BF7D\", \"#00B0F6\", \"#E76BF3\"], \"font\": {\"color\": \"rgb(51,51,51)\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"rgb(237,237,237)\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hovermode\": \"closest\", \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"rgb(237,237,237)\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"fillcolor\": \"black\", \"line\": {\"width\": 0}, \"opacity\": 0.3}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}}, \"title\": {\"text\": \"category\"}, \"xaxis\": {\"range\": [0, 159.6]}}, {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\"}); \n",
       "}\n",
       "</script><script type=\"text/javascript\">window.addEventListener(\"resize\", function(){if (document.getElementById(\"54cef0ef-2376-4d77-98bb-53d63d42eea6\")) {Plotly.Plots.resize(document.getElementById(\"54cef0ef-2376-4d77-98bb-53d63d42eea6\"));};})</script></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Category Coverage Difference (1 message(s)):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><div id=\"edf06c66-23c9-4da5-bcd1-2d391f55df60\" style=\"height: 900px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "if (document.getElementById(\"edf06c66-23c9-4da5-bcd1-2d391f55df60\")) {\n",
       "    Plotly.newPlot(\"edf06c66-23c9-4da5-bcd1-2d391f55df60\", [{\"name\": \"381798/1/2\", \"orientation\": \"h\", \"x\": [0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.002, 0.003, 0.003, 0.003, 0.004, 0.005, 0.005, 0.005, 0.006, 0.006, 0.006, 0.007, 0.007, 0.008, 0.009, 0.01, 0.011, 0.011, 0.011, 0.012, 0.013, 0.014, 0.016, 0.017, 0.017, 0.018, 0.019, 0.019, 0.026, 0.029, 0.03, 0.032, 0.035, 0.048, 0.054, 0.065, 0.067, 0.075, 0.11, 0.152], \"y\": [\"Novels\", \"Suspense\", \"Erotica\", \"Paranormal\", \"Academic\", \"Parenting\", \"Crime\", \"Cultural\", \"Adult Fiction\", \"Short Stories\", \"Historical\", \"Christian\", \"Politics\", \"Contemporary\", \"Health\", \"Sports and Games\", \"Biography\", \"Self Help\", \"New Adult\", \"Christian Fiction\", \"Spirituality\", \"Religion\", \"Psychology\", \"Art\", \"Autobiography\", \"Humor\", \"Thriller\", \"Philosophy\", \"Travel\", \"Business\", \"Music\", \"Science\", \"Science Fiction\", \"Womens Fiction\", \"Horror\", \"History\", \"Poetry\", \"Classics\", \"Historical Fiction\", \"Childrens\", \"Food and Drink\", \"Mystery\", \"Romance\", \"Fantasy\", \"Young Adult\", \"Fiction\", \"Add a comment\", \"Sequential Art\", \"Nonfiction\", \"Default\"], \"type\": \"bar\", \"uid\": \"380b344f-1f31-4625-9dc6-8ddc0d059637\"}, {\"name\": \"381798/1/1\", \"orientation\": \"h\", \"x\": [0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.002, 0.003, 0.003, 0.003, 0.004, 0.005, 0.005, 0.005, 0.006, 0.006, 0.006, 0.007, 0.007, 0.008, 0.009, 0.01, 0.011, 0.011, 0.011, 0.012, 0.013, 0.014, 0.016, 0.017, 0.017, 0.018, 0.019, 0.019, 0.026, 0.029, 0.03, 0.032, 0.035, 0.048, 0.054, 0.065, 0.067, 0.075, 0.11, 0.152], \"y\": [\"Novels\", \"Suspense\", \"Erotica\", \"Paranormal\", \"Academic\", \"Parenting\", \"Crime\", \"Cultural\", \"Adult Fiction\", \"Short Stories\", \"Historical\", \"Christian\", \"Politics\", \"Contemporary\", \"Health\", \"Sports and Games\", \"Biography\", \"Self Help\", \"New Adult\", \"Christian Fiction\", \"Spirituality\", \"Religion\", \"Psychology\", \"Art\", \"Autobiography\", \"Humor\", \"Thriller\", \"Philosophy\", \"Travel\", \"Business\", \"Music\", \"Science\", \"Science Fiction\", \"Womens Fiction\", \"Horror\", \"History\", \"Poetry\", \"Classics\", \"Historical Fiction\", \"Childrens\", \"Food and Drink\", \"Mystery\", \"Romance\", \"Fantasy\", \"Young Adult\", \"Fiction\", \"Add a comment\", \"Sequential Art\", \"Nonfiction\", \"Default\"], \"type\": \"bar\", \"uid\": \"322b9dcf-4e18-495e-a897-ce238343f8fb\"}], {\"bargap\": 0.1, \"height\": 900, \"hovermode\": \"y\", \"margin\": {\"l\": 200, \"t\": 35}, \"template\": {\"data\": {\"barpolar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"bar\": [{\"marker\": {\"line\": {\"color\": \"rgb(237,237,237)\", \"width\": 0.5}}, \"type\": \"bar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"baxis\": {\"endlinecolor\": \"rgb(51,51,51)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(51,51,51)\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"choropleth\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contourcarpet\"}], \"contour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"contour\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmapgl\"}], \"heatmap\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"heatmap\"}], \"histogram2dcontour\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2dcontour\"}], \"histogram2d\": [{\"autocolorscale\": true, \"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"histogram2d\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"histogram\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"parcoords\"}], \"scatter3d\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scattermapbox\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolargl\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterpolar\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatter\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"tickcolor\": \"rgb(237,237,237)\", \"ticklen\": 6, \"ticks\": \"inside\"}, \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"rgb(237,237,237)\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"rgb(217,217,217)\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"sequential\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]], \"sequentialminus\": [[0, \"rgb(20,44,66)\"], [1, \"rgb(90,179,244)\"]]}, \"colorway\": [\"#F8766D\", \"#A3A500\", \"#00BF7D\", \"#00B0F6\", \"#E76BF3\"], \"font\": {\"color\": \"rgb(51,51,51)\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"rgb(237,237,237)\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hovermode\": \"closest\", \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"rgb(237,237,237)\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"rgb(237,237,237)\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"fillcolor\": \"black\", \"line\": {\"width\": 0}, \"opacity\": 0.3}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}, \"bgcolor\": \"rgb(237,237,237)\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\"}}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"showgrid\": true, \"tickcolor\": \"rgb(51,51,51)\", \"ticks\": \"outside\", \"zerolinecolor\": \"white\"}}}, \"title\": {\"text\": \"Coverage for category\"}, \"xaxis\": {\"range\": [0, 1.05], \"tickformat\": \".2p\"}}, {\"showLink\": false, \"linkText\": \"Export to plot.ly\", \"plotlyServerURL\": \"https://plot.ly\"}); \n",
       "}\n",
       "</script><script type=\"text/javascript\">window.addEventListener(\"resize\", function(){if (document.getElementById(\"edf06c66-23c9-4da5-bcd1-2d391f55df60\")) {Plotly.Plots.resize(document.getElementById(\"edf06c66-23c9-4da5-bcd1-2d391f55df60\"));};})</script></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a.report_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_duplicates_by(a.source_items.df, [\"title\", \"price\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
