>>> import pandas as pd
>>> np = pd.np
>>> import os
>>> from twip.constant import *
>>> import gzip
>>> with gzip.open(os.path.join(DATA_PATH, 'text.csv.gz'), 'rb') as f:
...     df = pd.DataFrame.from_csv(f, encoding='utf8')
>>> df
                        id                                               text  \
87      724276510626979840  #python never stop learning what you enjoy doi...   
88      724276498249572352  Watching Boa vs. Python — https://t.co/Pivpk02s2A   
90      724276388325412866  Monty Python - The silly walk https://t.co/C0J...   
101     725078887005347840  Senior Software Engineer Full Stack Python Dja...   
102     725078874338541572  Architect Django Solr Platform Engineer With P...   
103     725078868802068481              peaceful rain? Python - inevitability   
104     725078695824740352  #How to make while loops in Python? [on hold]\...   
105     725078630750105600  #How to make while loops in Python? [on hold]\...   
106     725078507387293698  #How to make while loops in Python? [on hold]\...   
107     725078452928450560  RT @PythonWeekly: serialplot - A python 2.7 ap...   
108     725078446519492609  RT @nixcraft: Let me try this with ISP supplie...   
109     725078339875160064  Async GraphQL backend? https://t.co/KJn01x7ot4...   
110     725078303179214849  RT @larrykim: Which #programming language is r...   
111     725078302805909505  The Learn #Python NYC meetup is hosting the Su...   
112     725078208736022529  @abrightmoore could you run that code straight...   
113     725077917219323904  @icecolbeveridge I've just always imagined Pyt...   
114     725077831240265728  Python PE/Architect, QA/Performance Tools at S...   
115     725077738290331651  @AlbinoNagaBoy The python made sure he was per...   
116     725077721680760832  Challenge Your Python Knowledge https://t.co/f...   
117     725077705184710656  RT @abunchofdata: How To Extract Feature Vecto...   
118     725077402280427523  RT @kennethreitz: More interesting graphs! Pyt...   
119     725077376321900544  RT @DZRE_Giveaways: My First Giveaway Lets Hit...   
120     725077370764443648  Senior Python Django Engineer Jobs in San Fran...   
121     725077349025366018  Reptile Talk Episode 1: Carpet Python + Agains...   
122     725077318851543040  @ImLundy i used to be in pms. When python  i b...   
123     725077308822949888  RT @msisodia: All computers in India using lan...   
124     725077288094687236  Merge of more than 2 python pandas data frames...   
125     725077190698778624  97% off Learning Python for Data Analysis and ...   
126     725077136227315712  Don't forget tomorrow night...Python OOP and T...   
127     725077008498188288  We are hiring: Senior Software Development Eng...   
...                    ...                                                ...   
193348  731128258532540416  RT @randal_olson: Free video course on YouTube...   
193349  731127852318523393  RT @Petzoldt: Excellent hands-on tutorial abou...   
193350  731127380249444357  RT @nixcraft: Announcing Certbot: EFF's Client...   
193351  731127035058212865  Snake on a drain: Ball python crawls through p...   
193352  731126496237133824  Snake on a drain: Baby ball python crawls thro...   
193353  731125253557104640  $12 (40% off) – programming in python https://...   
193354  731125050250801156  Step by Step Python Programming Tutorial For B...   
193355  731124521634172928  I almost was gonna name him Karl 😭, hmm he a b...   
193356  731124376968429573  Display search results in JSON format using th...   
193357  731123657850900480  How to build a CRUD application using Flask (P...   
193358  731123538791276544  @HeidiHenricks I really wanted a banana ball p...   
193359  731122988599410688  dockhand - Multi docker image build management...   
193360  731122771569352704  RT @nixcraft: Announcing Certbot: EFF's Client...   
193361  731122539750006784  RT @svvitale: Careful if you're using #python,...   
193362  731122501049286657  RT @maher275: RSPET – Python Reverse Shell and...   
193363  731122251278499841  RT @javacodegeeks: Top Performance Metrics for...   
193364  724276118015086594  #intelmaker Intel Edison Board is a really gre...   
193365  724276114374426625  Fullstack Software Engineer Clojure Datomic Py...   
193366  724276097018265601  RT @pauljamesharper: Passed "Programming for E...   
193367  724276059764453376  thank god i know how to read + write python th...   
193368  724276000167739392  #python nexmo 1.1.0: Nexmo Client Library for ...   
193369  724275932538630144  Lanjut. Besok libur ini 😅😅 ★ Boa vs. Python (w...   
193370  724275869796151296  RT @LatestKruger: 17:29\nPython came down,now ...   
193371  724275850946961408  La idle de python en GNU/Linux Python es uno d...   
193372  724275847591546880  Las 3 mejores ides para Python Antes de empeza...   
193373  724275810777985026  Gagal tidur gegara habis vertical limit ada fi...   
193374  724275650043875328  Go boa wkwk💪😄 ★ Boa vs. Python — https://t.co/...   
193375  724275609858392066  RT @RealPython: List of Python API Wrappers &g...   
193376  724275578879111169  Watching Boa vs. Python — https://t.co/5THbrirfQO   
193377  724275568871673857  Чертова дюжина вакансий в IT и Digital /  / 1....   

                                                   tokens  \
87      [#python, never, stop, learning, what, you, en...   
88                        [Watching, Boa, vs., Python, —]   
90      [Monty, Python, -, The, silly, walk, via, @You...   
101     [Senior, Software, Engineer, Full, Stack, Pyth...   
102     [Architect, Django, Solr, Platform, Engineer, ...   
103           [peaceful, rain?, Python, -, inevitability]   
104     [#How, to, make, while, loops, in, Python?, [o...   
105     [#How, to, make, while, loops, in, Python?, [o...   
106     [#How, to, make, while, loops, in, Python?, [o...   
107     [RT, @PythonWeekly:, serialplot, -, A, python,...   
108     [RT, @nixcraft:, Let, me, try, this, with, ISP...   
109     [Async, GraphQL, backend?, #Python, I, was, wo...   
110     [RT, @larrykim:, Which, #programming, language...   
111     [The, Learn, #Python, NYC, meetup, is, hosting...   
112     [@abrightmoore, could, you, run, that, code, s...   
113     [@icecolbeveridge, I've, just, always, imagine...   
114     [Python, PE/Architect,, QA/Performance, Tools,...   
115     [@AlbinoNagaBoy, The, python, made, sure, he, ...   
116                   [Challenge, Your, Python Knowledge]   
117     [RT, @abunchofdata:, How, To, Extract, Feature...   
118     [RT, @kennethreitz:, More, interesting, graphs...   
119     [RT, @DZRE_Giveaways:, My, First, Giveaway, Le...   
120     [Senior, Python, Django, Engineer, Jobs, in, S...   
121     [Reptile, Talk, Episode, 1:, Carpet, Python, +...   
122     [@ImLundy, i, used, to, be, in, pms., When, py...   
123     [RT, @msisodia:, All, computers, in, India, us...   
124     [Merge, of, more, than, 2, python, pandas, dat...   
125     [97%, off, Learning, Python, for, Data, Analys...   
126     [Don't, forget, tomorrow, night...Python, OOP,...   
127     [We, are, hiring:, Senior, Software, Developme...   
...                                                   ...   
193348  [RT, @randal_olson:, Free, video, course, on, ...   
193349  [RT, @Petzoldt:, Excellent, hands-on, tutorial...   
193350  [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...   
193351  [Snake, on, a, drain:, Ball, python, crawls, t...   
193352  [Snake, on, a, drain:, Baby, ball, python, cra...   
193353  [$12, (40%, off), –, programming, in, python, ...   
193354  [Step, by, Step, Python, Programming, Tutorial...   
193355  [I, almost, was, gonna, name, him, Karl, 😭,, h...   
193356  [Display, search, results, in, JSON, format, u...   
193357  [How, to, build, a, CRUD, application, using, ...   
193358  [@HeidiHenricks, I, really, wanted, a, banana,...   
193359  [dockhand, -, Multi, docker, image, build, man...   
193360  [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...   
193361  [RT, @svvitale:, Careful, if, you're, using, #...   
193362  [RT, @maher275:, RSPET, –, Python, Reverse, Sh...   
193363  [RT, @javacodegeeks:, Top, Performance, Metric...   
193364  [#intelmaker, Intel, Edison, Board, is, a, rea...   
193365  [Fullstack, Software, Engineer, Clojure, Datom...   
193366  [RT, @pauljamesharper:, Passed, "Programming, ...   
193367  [thank, god, i, know, how, to, read, +, write,...   
193368  [#python, nexmo, 1.1.0:, Nexmo, Client, Librar...   
193369  [Lanjut., Besok, libur, ini, 😅😅, ★, Boa, vs., ...   
193370  [RT, @LatestKruger:, 17:29, Python, came, down...   
193371  [La, idle, de, python, en, GNU/Linux, Python, ...   
193372  [Las, 3, mejores, ides, para, Python, Antes, d...   
193373  [Gagal, tidur, gegara, habis, vertical, limit,...   
193374          [Go, boa, wkwk💪😄, ★, Boa, vs., Python, —]   
193375  [RT, @RealPython:, List, of, Python, API, Wrap...   
193376                    [Watching, Boa, vs., Python, —]   
193377  [Чертова, дюжина, вакансий, в, IT, и, Digital,...   

                                                      txt  
87       python never stop learning what you enjoy doing   
88                                Watching Boa vs Python   
90                Monty Python The silly walk via YouTube  
101     Senior Software Engineer Full Stack Python Dja...  
102     Architect Django Solr Platform Engineer With P...  
103                    peaceful rain Python inevitability  
104      How to make while loops in Python on hold Tec...  
105      How to make while loops in Python on hold Tec...  
106      How to make while loops in Python on hold How...  
107     RT PythonWeekly serialplot A python 2 7 applic...  
108     RT nixcraft Let me try this with ISP supplied ...  
109     Async GraphQL backend Python I was wondering i...  
110     RT larrykim Which programming language is righ...  
111     The Learn Python NYC meetup is hosting the Sun...  
112      abrightmoore could you run that code straight...  
113      icecolbeveridge I ve just always imagined Pyt...  
114     Python PE Architect QA Performance Tools at Sh...  
115      AlbinoNagaBoy The python made sure he was per...  
116                      Challenge Your Python Knowledge   
117     RT abunchofdata How To Extract Feature Vectors...  
118     RT kennethreitz More interesting graphs Python...  
119     RT DZRE_Giveaways My First Giveaway Lets Hit A...  
120     Senior Python Django Engineer Jobs in San Fran...  
121     Reptile Talk Episode 1 Carpet Python Against F...  
122      ImLundy i used to be in pms When python i bel...  
123     RT msisodia All computers in India using langu...  
124       Merge of more than 2 python pandas data frames   
125     97 off Learning Python for Data Analysis and V...  
126     Don t forget tomorrow night Python OOP and The...  
127     We are hiring Senior Software Development Engi...  
...                                                   ...  
193348  RT randal_olson Free video course on YouTube P...  
193349  RT Petzoldt Excellent hands on tutorial about ...  
193350  RT nixcraft Announcing Certbot EFF s Client fo...  
193351  Snake on a drain Ball python crawls through pi...  
193352  Snake on a drain Baby ball python crawls throu...  
193353   12 40 off programming in python DiscountsGlob...  
193354  Step by Step Python Programming Tutorial For B...  
193355  I almost was gonna name him Karl hmm he a ball...  
193356  Display search results in JSON format using th...  
193357  How to build a CRUD application using Flask Py...  
193358   HeidiHenricks I really wanted a banana ball p...  
193359  dockhand Multi docker image build management p...  
193360  RT nixcraft Announcing Certbot EFF s Client fo...  
193361  RT svvitale Careful if you re using python cel...  
193362  RT maher275 RSPET Python Reverse Shell and Pos...  
193363  RT javacodegeeks Top Performance Metrics for J...  
193364   intelmaker Intel Edison Board is a really gre...  
193365  Fullstack Software Engineer Clojure Datomic Py...  
193366  RT pauljamesharper Passed Programming for Ever...  
193367  thank god i know how to read write python than...  
193368   python nexmo 1 1 0 Nexmo Client Library for P...  
193369   Lanjut Besok libur ini Boa vs Python with Maiia   
193370  RT LatestKruger 17 29 Python came down now STA...  
193371  La idle de python en GNU Linux Python es uno d...  
193372  Las 3 mejores ides para Python Antes de empeza...  
193373  Gagal tidur gegara habis vertical limit ada fi...  
193374                         Go boa wkwk Boa vs Python   
193375  RT RealPython List of Python API Wrappers gt g...  
193376                            Watching Boa vs Python   
193377      IT Digital 1 Go 150 000 2 Senior Python Djan   

[183070 rows x 4 columns]
>>> df.txt
87         python never stop learning what you enjoy doing 
88                                  Watching Boa vs Python 
90                  Monty Python The silly walk via YouTube
101       Senior Software Engineer Full Stack Python Dja...
102       Architect Django Solr Platform Engineer With P...
103                      peaceful rain Python inevitability
104        How to make while loops in Python on hold Tec...
105        How to make while loops in Python on hold Tec...
106        How to make while loops in Python on hold How...
107       RT PythonWeekly serialplot A python 2 7 applic...
108       RT nixcraft Let me try this with ISP supplied ...
109       Async GraphQL backend Python I was wondering i...
110       RT larrykim Which programming language is righ...
111       The Learn Python NYC meetup is hosting the Sun...
112        abrightmoore could you run that code straight...
113        icecolbeveridge I ve just always imagined Pyt...
114       Python PE Architect QA Performance Tools at Sh...
115        AlbinoNagaBoy The python made sure he was per...
116                        Challenge Your Python Knowledge 
117       RT abunchofdata How To Extract Feature Vectors...
118       RT kennethreitz More interesting graphs Python...
119       RT DZRE_Giveaways My First Giveaway Lets Hit A...
120       Senior Python Django Engineer Jobs in San Fran...
121       Reptile Talk Episode 1 Carpet Python Against F...
122        ImLundy i used to be in pms When python i bel...
123       RT msisodia All computers in India using langu...
124         Merge of more than 2 python pandas data frames 
125       97 off Learning Python for Data Analysis and V...
126       Don t forget tomorrow night Python OOP and The...
127       We are hiring Senior Software Development Engi...
                                ...                        
193348    RT randal_olson Free video course on YouTube P...
193349    RT Petzoldt Excellent hands on tutorial about ...
193350    RT nixcraft Announcing Certbot EFF s Client fo...
193351    Snake on a drain Ball python crawls through pi...
193352    Snake on a drain Baby ball python crawls throu...
193353     12 40 off programming in python DiscountsGlob...
193354    Step by Step Python Programming Tutorial For B...
193355    I almost was gonna name him Karl hmm he a ball...
193356    Display search results in JSON format using th...
193357    How to build a CRUD application using Flask Py...
193358     HeidiHenricks I really wanted a banana ball p...
193359    dockhand Multi docker image build management p...
193360    RT nixcraft Announcing Certbot EFF s Client fo...
193361    RT svvitale Careful if you re using python cel...
193362    RT maher275 RSPET Python Reverse Shell and Pos...
193363    RT javacodegeeks Top Performance Metrics for J...
193364     intelmaker Intel Edison Board is a really gre...
193365    Fullstack Software Engineer Clojure Datomic Py...
193366    RT pauljamesharper Passed Programming for Ever...
193367    thank god i know how to read write python than...
193368     python nexmo 1 1 0 Nexmo Client Library for P...
193369     Lanjut Besok libur ini Boa vs Python with Maiia 
193370    RT LatestKruger 17 29 Python came down now STA...
193371    La idle de python en GNU Linux Python es uno d...
193372    Las 3 mejores ides para Python Antes de empeza...
193373    Gagal tidur gegara habis vertical limit ada fi...
193374                           Go boa wkwk Boa vs Python 
193375    RT RealPython List of Python API Wrappers gt g...
193376                              Watching Boa vs Python 
193377        IT Digital 1 Go 150 000 2 Senior Python Djan 
Name: txt, dtype: object
>>> df -h
>>> df
                        id                                               text  \
87      724276510626979840  #python never stop learning what you enjoy doi...   
88      724276498249572352  Watching Boa vs. Python — https://t.co/Pivpk02s2A   
90      724276388325412866  Monty Python - The silly walk https://t.co/C0J...   
101     725078887005347840  Senior Software Engineer Full Stack Python Dja...   
102     725078874338541572  Architect Django Solr Platform Engineer With P...   
103     725078868802068481              peaceful rain? Python - inevitability   
104     725078695824740352  #How to make while loops in Python? [on hold]\...   
105     725078630750105600  #How to make while loops in Python? [on hold]\...   
106     725078507387293698  #How to make while loops in Python? [on hold]\...   
107     725078452928450560  RT @PythonWeekly: serialplot - A python 2.7 ap...   
108     725078446519492609  RT @nixcraft: Let me try this with ISP supplie...   
109     725078339875160064  Async GraphQL backend? https://t.co/KJn01x7ot4...   
110     725078303179214849  RT @larrykim: Which #programming language is r...   
111     725078302805909505  The Learn #Python NYC meetup is hosting the Su...   
112     725078208736022529  @abrightmoore could you run that code straight...   
113     725077917219323904  @icecolbeveridge I've just always imagined Pyt...   
114     725077831240265728  Python PE/Architect, QA/Performance Tools at S...   
115     725077738290331651  @AlbinoNagaBoy The python made sure he was per...   
116     725077721680760832  Challenge Your Python Knowledge https://t.co/f...   
117     725077705184710656  RT @abunchofdata: How To Extract Feature Vecto...   
118     725077402280427523  RT @kennethreitz: More interesting graphs! Pyt...   
119     725077376321900544  RT @DZRE_Giveaways: My First Giveaway Lets Hit...   
120     725077370764443648  Senior Python Django Engineer Jobs in San Fran...   
121     725077349025366018  Reptile Talk Episode 1: Carpet Python + Agains...   
122     725077318851543040  @ImLundy i used to be in pms. When python  i b...   
123     725077308822949888  RT @msisodia: All computers in India using lan...   
124     725077288094687236  Merge of more than 2 python pandas data frames...   
125     725077190698778624  97% off Learning Python for Data Analysis and ...   
126     725077136227315712  Don't forget tomorrow night...Python OOP and T...   
127     725077008498188288  We are hiring: Senior Software Development Eng...   
...                    ...                                                ...   
193348  731128258532540416  RT @randal_olson: Free video course on YouTube...   
193349  731127852318523393  RT @Petzoldt: Excellent hands-on tutorial abou...   
193350  731127380249444357  RT @nixcraft: Announcing Certbot: EFF's Client...   
193351  731127035058212865  Snake on a drain: Ball python crawls through p...   
193352  731126496237133824  Snake on a drain: Baby ball python crawls thro...   
193353  731125253557104640  $12 (40% off) – programming in python https://...   
193354  731125050250801156  Step by Step Python Programming Tutorial For B...   
193355  731124521634172928  I almost was gonna name him Karl 😭, hmm he a b...   
193356  731124376968429573  Display search results in JSON format using th...   
193357  731123657850900480  How to build a CRUD application using Flask (P...   
193358  731123538791276544  @HeidiHenricks I really wanted a banana ball p...   
193359  731122988599410688  dockhand - Multi docker image build management...   
193360  731122771569352704  RT @nixcraft: Announcing Certbot: EFF's Client...   
193361  731122539750006784  RT @svvitale: Careful if you're using #python,...   
193362  731122501049286657  RT @maher275: RSPET – Python Reverse Shell and...   
193363  731122251278499841  RT @javacodegeeks: Top Performance Metrics for...   
193364  724276118015086594  #intelmaker Intel Edison Board is a really gre...   
193365  724276114374426625  Fullstack Software Engineer Clojure Datomic Py...   
193366  724276097018265601  RT @pauljamesharper: Passed "Programming for E...   
193367  724276059764453376  thank god i know how to read + write python th...   
193368  724276000167739392  #python nexmo 1.1.0: Nexmo Client Library for ...   
193369  724275932538630144  Lanjut. Besok libur ini 😅😅 ★ Boa vs. Python (w...   
193370  724275869796151296  RT @LatestKruger: 17:29\nPython came down,now ...   
193371  724275850946961408  La idle de python en GNU/Linux Python es uno d...   
193372  724275847591546880  Las 3 mejores ides para Python Antes de empeza...   
193373  724275810777985026  Gagal tidur gegara habis vertical limit ada fi...   
193374  724275650043875328  Go boa wkwk💪😄 ★ Boa vs. Python — https://t.co/...   
193375  724275609858392066  RT @RealPython: List of Python API Wrappers &g...   
193376  724275578879111169  Watching Boa vs. Python — https://t.co/5THbrirfQO   
193377  724275568871673857  Чертова дюжина вакансий в IT и Digital /  / 1....   

                                                   tokens  \
87      [#python, never, stop, learning, what, you, en...   
88                        [Watching, Boa, vs., Python, —]   
90      [Monty, Python, -, The, silly, walk, via, @You...   
101     [Senior, Software, Engineer, Full, Stack, Pyth...   
102     [Architect, Django, Solr, Platform, Engineer, ...   
103           [peaceful, rain?, Python, -, inevitability]   
104     [#How, to, make, while, loops, in, Python?, [o...   
105     [#How, to, make, while, loops, in, Python?, [o...   
106     [#How, to, make, while, loops, in, Python?, [o...   
107     [RT, @PythonWeekly:, serialplot, -, A, python,...   
108     [RT, @nixcraft:, Let, me, try, this, with, ISP...   
109     [Async, GraphQL, backend?, #Python, I, was, wo...   
110     [RT, @larrykim:, Which, #programming, language...   
111     [The, Learn, #Python, NYC, meetup, is, hosting...   
112     [@abrightmoore, could, you, run, that, code, s...   
113     [@icecolbeveridge, I've, just, always, imagine...   
114     [Python, PE/Architect,, QA/Performance, Tools,...   
115     [@AlbinoNagaBoy, The, python, made, sure, he, ...   
116                   [Challenge, Your, Python Knowledge]   
117     [RT, @abunchofdata:, How, To, Extract, Feature...   
118     [RT, @kennethreitz:, More, interesting, graphs...   
119     [RT, @DZRE_Giveaways:, My, First, Giveaway, Le...   
120     [Senior, Python, Django, Engineer, Jobs, in, S...   
121     [Reptile, Talk, Episode, 1:, Carpet, Python, +...   
122     [@ImLundy, i, used, to, be, in, pms., When, py...   
123     [RT, @msisodia:, All, computers, in, India, us...   
124     [Merge, of, more, than, 2, python, pandas, dat...   
125     [97%, off, Learning, Python, for, Data, Analys...   
126     [Don't, forget, tomorrow, night...Python, OOP,...   
127     [We, are, hiring:, Senior, Software, Developme...   
...                                                   ...   
193348  [RT, @randal_olson:, Free, video, course, on, ...   
193349  [RT, @Petzoldt:, Excellent, hands-on, tutorial...   
193350  [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...   
193351  [Snake, on, a, drain:, Ball, python, crawls, t...   
193352  [Snake, on, a, drain:, Baby, ball, python, cra...   
193353  [$12, (40%, off), –, programming, in, python, ...   
193354  [Step, by, Step, Python, Programming, Tutorial...   
193355  [I, almost, was, gonna, name, him, Karl, 😭,, h...   
193356  [Display, search, results, in, JSON, format, u...   
193357  [How, to, build, a, CRUD, application, using, ...   
193358  [@HeidiHenricks, I, really, wanted, a, banana,...   
193359  [dockhand, -, Multi, docker, image, build, man...   
193360  [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...   
193361  [RT, @svvitale:, Careful, if, you're, using, #...   
193362  [RT, @maher275:, RSPET, –, Python, Reverse, Sh...   
193363  [RT, @javacodegeeks:, Top, Performance, Metric...   
193364  [#intelmaker, Intel, Edison, Board, is, a, rea...   
193365  [Fullstack, Software, Engineer, Clojure, Datom...   
193366  [RT, @pauljamesharper:, Passed, "Programming, ...   
193367  [thank, god, i, know, how, to, read, +, write,...   
193368  [#python, nexmo, 1.1.0:, Nexmo, Client, Librar...   
193369  [Lanjut., Besok, libur, ini, 😅😅, ★, Boa, vs., ...   
193370  [RT, @LatestKruger:, 17:29, Python, came, down...   
193371  [La, idle, de, python, en, GNU/Linux, Python, ...   
193372  [Las, 3, mejores, ides, para, Python, Antes, d...   
193373  [Gagal, tidur, gegara, habis, vertical, limit,...   
193374          [Go, boa, wkwk💪😄, ★, Boa, vs., Python, —]   
193375  [RT, @RealPython:, List, of, Python, API, Wrap...   
193376                    [Watching, Boa, vs., Python, —]   
193377  [Чертова, дюжина, вакансий, в, IT, и, Digital,...   

                                                      txt  
87       python never stop learning what you enjoy doing   
88                                Watching Boa vs Python   
90                Monty Python The silly walk via YouTube  
101     Senior Software Engineer Full Stack Python Dja...  
102     Architect Django Solr Platform Engineer With P...  
103                    peaceful rain Python inevitability  
104      How to make while loops in Python on hold Tec...  
105      How to make while loops in Python on hold Tec...  
106      How to make while loops in Python on hold How...  
107     RT PythonWeekly serialplot A python 2 7 applic...  
108     RT nixcraft Let me try this with ISP supplied ...  
109     Async GraphQL backend Python I was wondering i...  
110     RT larrykim Which programming language is righ...  
111     The Learn Python NYC meetup is hosting the Sun...  
112      abrightmoore could you run that code straight...  
113      icecolbeveridge I ve just always imagined Pyt...  
114     Python PE Architect QA Performance Tools at Sh...  
115      AlbinoNagaBoy The python made sure he was per...  
116                      Challenge Your Python Knowledge   
117     RT abunchofdata How To Extract Feature Vectors...  
118     RT kennethreitz More interesting graphs Python...  
119     RT DZRE_Giveaways My First Giveaway Lets Hit A...  
120     Senior Python Django Engineer Jobs in San Fran...  
121     Reptile Talk Episode 1 Carpet Python Against F...  
122      ImLundy i used to be in pms When python i bel...  
123     RT msisodia All computers in India using langu...  
124       Merge of more than 2 python pandas data frames   
125     97 off Learning Python for Data Analysis and V...  
126     Don t forget tomorrow night Python OOP and The...  
127     We are hiring Senior Software Development Engi...  
...                                                   ...  
193348  RT randal_olson Free video course on YouTube P...  
193349  RT Petzoldt Excellent hands on tutorial about ...  
193350  RT nixcraft Announcing Certbot EFF s Client fo...  
193351  Snake on a drain Ball python crawls through pi...  
193352  Snake on a drain Baby ball python crawls throu...  
193353   12 40 off programming in python DiscountsGlob...  
193354  Step by Step Python Programming Tutorial For B...  
193355  I almost was gonna name him Karl hmm he a ball...  
193356  Display search results in JSON format using th...  
193357  How to build a CRUD application using Flask Py...  
193358   HeidiHenricks I really wanted a banana ball p...  
193359  dockhand Multi docker image build management p...  
193360  RT nixcraft Announcing Certbot EFF s Client fo...  
193361  RT svvitale Careful if you re using python cel...  
193362  RT maher275 RSPET Python Reverse Shell and Pos...  
193363  RT javacodegeeks Top Performance Metrics for J...  
193364   intelmaker Intel Edison Board is a really gre...  
193365  Fullstack Software Engineer Clojure Datomic Py...  
193366  RT pauljamesharper Passed Programming for Ever...  
193367  thank god i know how to read write python than...  
193368   python nexmo 1 1 0 Nexmo Client Library for P...  
193369   Lanjut Besok libur ini Boa vs Python with Maiia   
193370  RT LatestKruger 17 29 Python came down now STA...  
193371  La idle de python en GNU Linux Python es uno d...  
193372  Las 3 mejores ides para Python Antes de empeza...  
193373  Gagal tidur gegara habis vertical limit ada fi...  
193374                         Go boa wkwk Boa vs Python   
193375  RT RealPython List of Python API Wrappers gt g...  
193376                            Watching Boa vs Python   
193377      IT Digital 1 Go 150 000 2 Senior Python Djan   

[183070 rows x 4 columns]
>>> !df
>>> !df -h
>>> from gensim.models import Dictionary, TfidfModel
>>> from gensim.models import TfidfModel
>>> from gensim.corpora imoprt Dictionary
>>> from gensim.corpora import Dictionary
>>> Dictionary.from_corpus?
>>> Dictionary.from_documents?
>>> Dictionary.from_documents??
>>> df.tokens
87        [#python, never, stop, learning, what, you, en...
88                          [Watching, Boa, vs., Python, —]
90        [Monty, Python, -, The, silly, walk, via, @You...
101       [Senior, Software, Engineer, Full, Stack, Pyth...
102       [Architect, Django, Solr, Platform, Engineer, ...
103             [peaceful, rain?, Python, -, inevitability]
104       [#How, to, make, while, loops, in, Python?, [o...
105       [#How, to, make, while, loops, in, Python?, [o...
106       [#How, to, make, while, loops, in, Python?, [o...
107       [RT, @PythonWeekly:, serialplot, -, A, python,...
108       [RT, @nixcraft:, Let, me, try, this, with, ISP...
109       [Async, GraphQL, backend?, #Python, I, was, wo...
110       [RT, @larrykim:, Which, #programming, language...
111       [The, Learn, #Python, NYC, meetup, is, hosting...
112       [@abrightmoore, could, you, run, that, code, s...
113       [@icecolbeveridge, I've, just, always, imagine...
114       [Python, PE/Architect,, QA/Performance, Tools,...
115       [@AlbinoNagaBoy, The, python, made, sure, he, ...
116                     [Challenge, Your, Python Knowledge]
117       [RT, @abunchofdata:, How, To, Extract, Feature...
118       [RT, @kennethreitz:, More, interesting, graphs...
119       [RT, @DZRE_Giveaways:, My, First, Giveaway, Le...
120       [Senior, Python, Django, Engineer, Jobs, in, S...
121       [Reptile, Talk, Episode, 1:, Carpet, Python, +...
122       [@ImLundy, i, used, to, be, in, pms., When, py...
123       [RT, @msisodia:, All, computers, in, India, us...
124       [Merge, of, more, than, 2, python, pandas, dat...
125       [97%, off, Learning, Python, for, Data, Analys...
126       [Don't, forget, tomorrow, night...Python, OOP,...
127       [We, are, hiring:, Senior, Software, Developme...
                                ...                        
193348    [RT, @randal_olson:, Free, video, course, on, ...
193349    [RT, @Petzoldt:, Excellent, hands-on, tutorial...
193350    [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...
193351    [Snake, on, a, drain:, Ball, python, crawls, t...
193352    [Snake, on, a, drain:, Baby, ball, python, cra...
193353    [$12, (40%, off), –, programming, in, python, ...
193354    [Step, by, Step, Python, Programming, Tutorial...
193355    [I, almost, was, gonna, name, him, Karl, 😭,, h...
193356    [Display, search, results, in, JSON, format, u...
193357    [How, to, build, a, CRUD, application, using, ...
193358    [@HeidiHenricks, I, really, wanted, a, banana,...
193359    [dockhand, -, Multi, docker, image, build, man...
193360    [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...
193361    [RT, @svvitale:, Careful, if, you're, using, #...
193362    [RT, @maher275:, RSPET, –, Python, Reverse, Sh...
193363    [RT, @javacodegeeks:, Top, Performance, Metric...
193364    [#intelmaker, Intel, Edison, Board, is, a, rea...
193365    [Fullstack, Software, Engineer, Clojure, Datom...
193366    [RT, @pauljamesharper:, Passed, "Programming, ...
193367    [thank, god, i, know, how, to, read, +, write,...
193368    [#python, nexmo, 1.1.0:, Nexmo, Client, Librar...
193369    [Lanjut., Besok, libur, ini, 😅😅, ★, Boa, vs., ...
193370    [RT, @LatestKruger:, 17:29, Python, came, down...
193371    [La, idle, de, python, en, GNU/Linux, Python, ...
193372    [Las, 3, mejores, ides, para, Python, Antes, d...
193373    [Gagal, tidur, gegara, habis, vertical, limit,...
193374            [Go, boa, wkwk💪😄, ★, Boa, vs., Python, —]
193375    [RT, @RealPython:, List, of, Python, API, Wrap...
193376                      [Watching, Boa, vs., Python, —]
193377    [Чертова, дюжина, вакансий, в, IT, и, Digital,...
Name: tokens, dtype: object
>>> d = Dictionary.from_documents(df.tokens)
>>> df.tokens[0]
>>> df.tokens
87        [#python, never, stop, learning, what, you, en...
88                          [Watching, Boa, vs., Python, —]
90        [Monty, Python, -, The, silly, walk, via, @You...
101       [Senior, Software, Engineer, Full, Stack, Pyth...
102       [Architect, Django, Solr, Platform, Engineer, ...
103             [peaceful, rain?, Python, -, inevitability]
104       [#How, to, make, while, loops, in, Python?, [o...
105       [#How, to, make, while, loops, in, Python?, [o...
106       [#How, to, make, while, loops, in, Python?, [o...
107       [RT, @PythonWeekly:, serialplot, -, A, python,...
108       [RT, @nixcraft:, Let, me, try, this, with, ISP...
109       [Async, GraphQL, backend?, #Python, I, was, wo...
110       [RT, @larrykim:, Which, #programming, language...
111       [The, Learn, #Python, NYC, meetup, is, hosting...
112       [@abrightmoore, could, you, run, that, code, s...
113       [@icecolbeveridge, I've, just, always, imagine...
114       [Python, PE/Architect,, QA/Performance, Tools,...
115       [@AlbinoNagaBoy, The, python, made, sure, he, ...
116                     [Challenge, Your, Python Knowledge]
117       [RT, @abunchofdata:, How, To, Extract, Feature...
118       [RT, @kennethreitz:, More, interesting, graphs...
119       [RT, @DZRE_Giveaways:, My, First, Giveaway, Le...
120       [Senior, Python, Django, Engineer, Jobs, in, S...
121       [Reptile, Talk, Episode, 1:, Carpet, Python, +...
122       [@ImLundy, i, used, to, be, in, pms., When, py...
123       [RT, @msisodia:, All, computers, in, India, us...
124       [Merge, of, more, than, 2, python, pandas, dat...
125       [97%, off, Learning, Python, for, Data, Analys...
126       [Don't, forget, tomorrow, night...Python, OOP,...
127       [We, are, hiring:, Senior, Software, Developme...
                                ...                        
193348    [RT, @randal_olson:, Free, video, course, on, ...
193349    [RT, @Petzoldt:, Excellent, hands-on, tutorial...
193350    [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...
193351    [Snake, on, a, drain:, Ball, python, crawls, t...
193352    [Snake, on, a, drain:, Baby, ball, python, cra...
193353    [$12, (40%, off), –, programming, in, python, ...
193354    [Step, by, Step, Python, Programming, Tutorial...
193355    [I, almost, was, gonna, name, him, Karl, 😭,, h...
193356    [Display, search, results, in, JSON, format, u...
193357    [How, to, build, a, CRUD, application, using, ...
193358    [@HeidiHenricks, I, really, wanted, a, banana,...
193359    [dockhand, -, Multi, docker, image, build, man...
193360    [RT, @nixcraft:, Announcing, Certbot:, EFF's, ...
193361    [RT, @svvitale:, Careful, if, you're, using, #...
193362    [RT, @maher275:, RSPET, –, Python, Reverse, Sh...
193363    [RT, @javacodegeeks:, Top, Performance, Metric...
193364    [#intelmaker, Intel, Edison, Board, is, a, rea...
193365    [Fullstack, Software, Engineer, Clojure, Datom...
193366    [RT, @pauljamesharper:, Passed, "Programming, ...
193367    [thank, god, i, know, how, to, read, +, write,...
193368    [#python, nexmo, 1.1.0:, Nexmo, Client, Librar...
193369    [Lanjut., Besok, libur, ini, 😅😅, ★, Boa, vs., ...
193370    [RT, @LatestKruger:, 17:29, Python, came, down...
193371    [La, idle, de, python, en, GNU/Linux, Python, ...
193372    [Las, 3, mejores, ides, para, Python, Antes, d...
193373    [Gagal, tidur, gegara, habis, vertical, limit,...
193374            [Go, boa, wkwk💪😄, ★, Boa, vs., Python, —]
193375    [RT, @RealPython:, List, of, Python, API, Wrap...
193376                      [Watching, Boa, vs., Python, —]
193377    [Чертова, дюжина, вакансий, в, IT, и, Digital,...
Name: tokens, dtype: object
>>> df.tokens.iloc[0]
u'[#python, never, stop, learning, what, you, enjoy, doing.]'
>>> df['tokens'] = df.txt.str.split()
>>> df.tokens
87        [python, never, stop, learning, what, you, enj...
88                              [Watching, Boa, vs, Python]
90          [Monty, Python, The, silly, walk, via, YouTube]
101       [Senior, Software, Engineer, Full, Stack, Pyth...
102       [Architect, Django, Solr, Platform, Engineer, ...
103                 [peaceful, rain, Python, inevitability]
104       [How, to, make, while, loops, in, Python, on, ...
105       [How, to, make, while, loops, in, Python, on, ...
106       [How, to, make, while, loops, in, Python, on, ...
107       [RT, PythonWeekly, serialplot, A, python, 2, 7...
108       [RT, nixcraft, Let, me, try, this, with, ISP, ...
109       [Async, GraphQL, backend, Python, I, was, wond...
110       [RT, larrykim, Which, programming, language, i...
111       [The, Learn, Python, NYC, meetup, is, hosting,...
112       [abrightmoore, could, you, run, that, code, st...
113       [icecolbeveridge, I, ve, just, always, imagine...
114       [Python, PE, Architect, QA, Performance, Tools...
115       [AlbinoNagaBoy, The, python, made, sure, he, w...
116                    [Challenge, Your, Python, Knowledge]
117       [RT, abunchofdata, How, To, Extract, Feature, ...
118       [RT, kennethreitz, More, interesting, graphs, ...
119       [RT, DZRE_Giveaways, My, First, Giveaway, Lets...
120       [Senior, Python, Django, Engineer, Jobs, in, S...
121       [Reptile, Talk, Episode, 1, Carpet, Python, Ag...
122       [ImLundy, i, used, to, be, in, pms, When, pyth...
123       [RT, msisodia, All, computers, in, India, usin...
124       [Merge, of, more, than, 2, python, pandas, dat...
125       [97, off, Learning, Python, for, Data, Analysi...
126       [Don, t, forget, tomorrow, night, Python, OOP,...
127       [We, are, hiring, Senior, Software, Developmen...
                                ...                        
193348    [RT, randal_olson, Free, video, course, on, Yo...
193349    [RT, Petzoldt, Excellent, hands, on, tutorial,...
193350    [RT, nixcraft, Announcing, Certbot, EFF, s, Cl...
193351    [Snake, on, a, drain, Ball, python, crawls, th...
193352    [Snake, on, a, drain, Baby, ball, python, craw...
193353    [12, 40, off, programming, in, python, Discoun...
193354    [Step, by, Step, Python, Programming, Tutorial...
193355    [I, almost, was, gonna, name, him, Karl, hmm, ...
193356    [Display, search, results, in, JSON, format, u...
193357    [How, to, build, a, CRUD, application, using, ...
193358    [HeidiHenricks, I, really, wanted, a, banana, ...
193359    [dockhand, Multi, docker, image, build, manage...
193360    [RT, nixcraft, Announcing, Certbot, EFF, s, Cl...
193361    [RT, svvitale, Careful, if, you, re, using, py...
193362    [RT, maher275, RSPET, Python, Reverse, Shell, ...
193363    [RT, javacodegeeks, Top, Performance, Metrics,...
193364    [intelmaker, Intel, Edison, Board, is, a, real...
193365    [Fullstack, Software, Engineer, Clojure, Datom...
193366    [RT, pauljamesharper, Passed, Programming, for...
193367    [thank, god, i, know, how, to, read, write, py...
193368    [python, nexmo, 1, 1, 0, Nexmo, Client, Librar...
193369    [Lanjut, Besok, libur, ini, Boa, vs, Python, w...
193370    [RT, LatestKruger, 17, 29, Python, came, down,...
193371    [La, idle, de, python, en, GNU, Linux, Python,...
193372    [Las, 3, mejores, ides, para, Python, Antes, d...
193373    [Gagal, tidur, gegara, habis, vertical, limit,...
193374                     [Go, boa, wkwk, Boa, vs, Python]
193375    [RT, RealPython, List, of, Python, API, Wrappe...
193376                          [Watching, Boa, vs, Python]
193377    [IT, Digital, 1, Go, 150, 000, 2, Senior, Pyth...
Name: tokens, dtype: object
>>> df.tokens.iloc[0]
[u'python',
 u'never',
 u'stop',
 u'learning',
 u'what',
 u'you',
 u'enjoy',
 u'doing']
>>> d = Dictionary.from_documents(df.txt.str.split())
>>> len(d)
91222
>>> TfidfModel?
>>> tfidf = TfidfModel(d)
>>> tfidf = TfidfModel(dictionary=d)
>>> tfidf
<gensim.models.tfidfmodel.TfidfModel at 0x7f6e74a8d890>
>>> len(tfidf)
>>> df.tokens[0]
>>> df.tokens.iloc[0]
[u'python',
 u'never',
 u'stop',
 u'learning',
 u'what',
 u'you',
 u'enjoy',
 u'doing']
>>> tfidf[df.tokens.iloc[0]]
>>> d.doc2bow(df.tokens.iloc[0])
[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)]
>>> d.doc2bow(df.tokens.iloc[1])
[(8, 1), (9, 1), (10, 1), (11, 1)]
>>> d.doc2bow(df.tokens.iloc[2])
[(8, 1), (12, 1), (13, 1), (14, 1), (15, 1), (16, 1), (17, 1)]
>>> df.tokens.iloc[0:2]
87    [python, never, stop, learning, what, you, enj...
88                          [Watching, Boa, vs, Python]
Name: tokens, dtype: object
>>> df.tokens.iloc[0:3]
87    [python, never, stop, learning, what, you, enj...
88                          [Watching, Boa, vs, Python]
90      [Monty, Python, The, silly, walk, via, YouTube]
Name: tokens, dtype: object
>>> d['python']
>>> d.token2id('python')
>>> d.token2id['python']
2
>>> d.token2id['you']
7
>>> d.id2token[0]