Pdf_crawl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 0\n",
    "literature = []\n",
    "\n",
    "with open(\"bill2020.txt\", \"r\") as f:\n",
    "    f1 = f.readlines()\n",
    "    #print(f1[4])\n",
    "    for i in enumerate(f1):\n",
    "        for j in enumerate(i[1]):            \n",
    "            if j[1] == \"$\":\n",
    "                #print(i)\n",
    "\n",
    "                literature.append(np.array((i[0],obtain_sentence(f1,i[1],i[0],j[0]))))\n",
    "\n",
    "                a += 1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"summary_text.txt\", \"+w\") as f:\n",
    "    for i in range(len(literature)):\n",
    "        f.write(np.array(literature)[:,1][i][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2 as pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "448"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(literature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    }
   ],
   "source": [
    "with open(\"EM_HW4.pdf\", \"rb\") as f:\n",
    "    pdfReader = pdf.PdfFileReader(f)\n",
    "    if pdfReader.isEncrypted:\n",
    "        pdfReader.decrypt('')\n",
    "    print(pdfReader.numPages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['b', 'c', 'd']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = \"abcd\"\n",
    "line = []\n",
    "for i in string:\n",
    "    line.append(i)\n",
    "\n",
    "line[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def obtain_sentence(full_doc,readline, row_index, char_index):\n",
    "    line_array = [i for i in readline]\n",
    "\n",
    "    dollar_amount_list = []\n",
    "    dollar_amount = ''\n",
    "    sentence_list = []\n",
    "    sentence = ''\n",
    "    sentence_list2 = []\n",
    "    sentence2 = ''\n",
    "    for i in line_array[char_index:]:\n",
    "        if i.isalpha():\n",
    "            break\n",
    "        dollar_amount_list.append(i)\n",
    "    for i in dollar_amount_list[:-1]:\n",
    "        dollar_amount+=i\n",
    "    \n",
    "    sentence_list = recursive_punc_find(full_doc, row_index, char_index, sentence_list, 1,0)\n",
    "    #print('done1')\n",
    "    sentence_list2 = recursive_punc_find(full_doc,row_index,len(line_array)-char_index,sentence_list2,-1,0)\n",
    "    #print(sentence_list2)\n",
    "                \n",
    "    \n",
    "    \n",
    "    for i in sentence_list2:\n",
    "        sentence += i\n",
    "        #print(i)\n",
    "    for i in sentence_list:\n",
    "        sentence += i\n",
    "    return dollar_amount, sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sentence = \"i want $1,000 immediately!\"\n",
    "char_index = 0\n",
    "for i in enumerate(test_sentence):\n",
    "    if i[1] == '$':\n",
    "        char_index=i[0]\n",
    "for i in enumerate(test_sentence[::-1]):\n",
    "    if i[1] == '$':\n",
    "        char_index=i[0]        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recursive_punc_find(full_doc, row_index, char_index, sentence, direction, recurslevel):\n",
    "    for i in full_doc[row_index][::direction][char_index:]:\n",
    "        sentence.append(i)\n",
    "\n",
    "        if i == '.':\n",
    "            #print(i)\n",
    "            return sentence[::direction]\n",
    "    #print(sentence)\n",
    "    if recurslevel < 2:\n",
    "        return recursive_punc_find(full_doc, row_index+direction, 0, sentence, direction,recurslevel+1)\n",
    "    else:\n",
    "        return sentence[::direction]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('', 'i want $1,000 immediately!')"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obtain_sentence(test_sentence, 0, char_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in stringlist[:-1]:\n",
    "    string+=i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'$1,000'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string[::1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in line_array[char_index:]:\n",
    "        sentence_list.append(i)\n",
    "        if i == '.':\n",
    "            break\n",
    "    if sentence_list[-1] != '.':\n",
    "        for i in full_doc[row_index+1]:\n",
    "            sentence_list.append(i)\n",
    "            if i == '.':\n",
    "                break\n",
    "                \n",
    "for i in line_array[::-1][(len(line_array)-char_index):]:\n",
    "        sentence_list2.append(i)\n",
    "        if i == '.':\n",
    "            break\n",
    "    if sentence_list2 != '.':\n",
    "        for i in full_doc[row_index-1][::-1]:\n",
    "            sentence_list2.append(i)\n",
    "            if i =='.':\n",
    "                break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}