From 3cff05035b222008d27601c0cc34e0597408b0be Mon Sep 17 00:00:00 2001
From: Master <bagira@forge.silogroup.lan>
Date: Sun, 2 Aug 2020 23:37:05 -0400
Subject: [PATCH] updated to reflect TPB site changes and various bug fixes

---
 Scallywag                                     |   2 +-
 config.ini                                    |   2 +-
 .../__pycache__/proxylister.cpython-37.pyc    | Bin 1647 -> 1783 bytes
 .../__pycache__/searcher.cpython-37.pyc       | Bin 3590 -> 4113 bytes
 rsrc/Scraper/proxylister.py                   |   6 ++-
 rsrc/Scraper/searcher.py                      |  45 +++++++++++++-----
 6 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/Scallywag b/Scallywag
index 368011a..eb13578 100755
--- a/Scallywag
+++ b/Scallywag
@@ -69,7 +69,7 @@ class Scallywag:
         self.searcher = searcher.Scraper( self.config )
 
         results =  self.searcher.get_results( search_terms )
-
+        self.status("{0} results found.".format( len(results) ) )
         for result in results:
             yield result
 
diff --git a/config.ini b/config.ini
index a3b3b8e..00d56e2 100644
--- a/config.ini
+++ b/config.ini
@@ -1,2 +1,2 @@
 [list_source]
-piratebay_proxy_list = thepiratebay-proxylist.se
+piratebay_proxy_list = https://piratebayproxy.info/
diff --git a/rsrc/Scraper/__pycache__/proxylister.cpython-37.pyc b/rsrc/Scraper/__pycache__/proxylister.cpython-37.pyc
index 772e9b96dcb200e731178831a225975db5606e62..59d387321484357a90281736d16a4e7be94cb56d 100644
GIT binary patch
delta 444
zcmaFQ^PQL1iI<m)0SK<=s>d(e$h(A5TLs9gVaQ@wzz8InvX~b#)`D0p3s_SaYZ$ZG
z7BbZ`)i4D!XfpXte#O|%D6qMRiI0(w1*nY`h@FAB*kp1avv_@lR1K30L#$9Oa|v?_
z(*l+npjpj~j0}ZR;S69CnEk3M_4SkTQ!1k!GE;1oauYN20us|xm16Z%GRvYJl5-M^
zi*1#X^Yco8GO0x{agbV&U}|0oP^cs^DJK<TKyhkfQF2C5YH?{!iIO=`sz|>i1#V7p
zW(mmXME#P~iV_XYDiO!B#LOI!ZiRrN{EA8|g=&LZP1ecxnI!~ou@)3%=9S!HOUp0H
zO)Sx5nf#MknH?kwR5)3NMG_KhOk9j0D74v*g@sX466p6L4miOHVu3<eadI@Pu7D_r
zD+VIO!4i`l*(4@!VC9up0CJ0&fCL9)kra^f)8r^ZQat$%s~n>gP__7EAvRG)fyo+d
F?f}e;Z<7E3

delta 322
zcmey)`<{o_iI<m)0SGQ%dJx07k#`BBr3R2!!;r<WfH8#;NHb+Ii!&@_tOc=I7O<u;
z)i7qUEo7=?s$mLd&}8<j;>ai|DJZtm*Pnccv7J$5a|#n5BOf!+5D<0-;$n%(4b0+J
zVD(IXRr>n+C5cHnsnHI}If=!^wn_y>`4yR|#Y(aIB}w@ymHH(``VJ|HC5gHz`MHUi
zd78|VeOV-!fK(Za^5g|9lALTnLm8O37`ZkdV_{*G6a(^#IN$^)hz0VK{N%^1y4*q_
zt}uuYnJmqwWhVz@7Bc|}4#pyJAmyjYQG}!hq!6UJNB~6e0f}3jt`*6t1tppJc}3!r
S71+cl&tem0<eR*K%^d)YqD9sK

diff --git a/rsrc/Scraper/__pycache__/searcher.cpython-37.pyc b/rsrc/Scraper/__pycache__/searcher.cpython-37.pyc
index 3b154d62a2e77fe5d3253c2c328ec3d691912af5..09a1533df4cfababfca9eba811a716d4a05e9188 100644
GIT binary patch
delta 1814
zcmZ8h-EUMy6rZpAxp$YP6k1GMOZmLCbP-DwQv?MfL?{YIa7nXScZRmhcDtQBC8ge_
zF-2dD4`g0_*uMB;VnTfI!GA#i0i*XHFg)_&qw$>Ex)66WzdPs7+;h(S=KRjrBTvUm
z?-z?9f!{A*9k03Nr=?-~$<yPNBa%qka;xqKL~<#f5Gl4;)dS0sE?6#D{xiKmX<#$@
z58Z43WSaW+I{%xF+Dl^ch6g?`!APb7QZA`vl7Gj$LW&+qY03<3vK~*xHr?i3$B5L~
zqLif0Z4x_kn!;Dl8vrrS=&JoqOi;(Z5UZsL^yvc-7V!hN<GjA<f`Mnq!?j?Nk{+a=
zk|u;m(LH-q28;JZ%C>2j-y!W8L5$OMo8E&^e@aPLP_i*#{8X5r8A|poqvQ?&5@6-A
zxJy1GZFU#@HVeH-@{~7=+pO#4$%@Un%vm(1lf*O=_m^5Ki?mACS|+KrmJ$<}^fAau
zzlLxa;UGZ97nIRMI1AN!Lv_s3N;}EQx76yIO3ao2Ug<;m=HyY5xrvcV>&*SAt`h~%
zq@n8_l}&n$m9|lB6T3P>|3?rC_+9b7GL6OBZaNuN_FebT;3O<1B>+MP7^7u6NXPB6
zduY>vSA8O@VC1xs9t9F<m+jKVn!G81w#z^p&uMc~Zh%TjU!jdZ=C9D^_X5dM)(n9*
zkqWqDJ{MDc0%Bx^<@!QfnOf`SQu{$%$m!81VI`+WW43w<0jbcZ5U{3n1>rQp8vwDh
zb9g+rK7;V49r2Fsdkdxs(n9F&QK|jdo1(M!u~!*TTzb-fLbhntlL4d^+Mm4JLuX-o
zEc5|*?1YJNcGiyiryPv<p1tW$mykjXm)G6z_8Wg}^T(83CiCBSg%M4s=cYn3ER?>*
z!E*bS0L$xJ4p@HQa={9W*Yl;91~3oPP(MjS;L(S>DD?N?T4X{Q_6jC?J9Sbw_0mEb
zrT&(?O>;@{UeQGVU!i1m{0<1SSK21sqMZqDdK;0E_h#np&%tbUFHFzOSMEb)Rg79U
zC*qC5?WGMheqws^LOk^x^|^8M<)(WV8?AbhoS%@&TvkbAV!m>+|KY7%4)2VAak?At
zv2Hka{Ux<oai@%Hi>lRus%Bb2WvjYwEN0GvURmqx-1{g5)Eppl>z$5j%ZztcIv}od
z_G)--3Gbm=P9VeDTDzlH8Y)SY%;>TX@I$QXjJN9#brH=R@cJ}f>@^q(iiWU|hRg*l
z(GtrCi<p3SB{4z;9Tq;bFT$}+hMVDs`Cu=~D*y}Q(E#=yl#8SXNiLaT=rk1Ma*`7K
zOV(w(-Gt>b$Xi`TPsdE32M<|ztu5916|Glv%=Hy`Rdv@pb+fpd@rU&m=6>|m6I_5D
z*z^)SBLO`RK=vWs7_l{fzqd=PEfp3=XJI{$0e;WHyc)DXakWl;K|yUl3->u#_s0HE
zn1WXRM`5g-t1GX4i~^fzG(3d5y9gui!aj}s=y2rgik*(Wq}OvddtkL|e~muBn&LiW
z%cuGVJm%t1?}MivMA(mjn|C_o49-xmx#qIqWfQ!Cfj>k!j=;#}>{H+I13&VE(rjK$
taj?@)MQeSzr7@-+L%4`gMmU9lSEdhuzajfqanAr20i1z?JN&cK;J@{WhgSdq

delta 1392
zcmZux&2Jk;6yKTIkMVlFaY$$q8j=(>G)odYR6&J+K!hTMT9rycLQ08@@oxQrV>`1G
zDY#=rB8Wo|NYPw5*f)g4fxm$R7cRhUPtXHW{{fDu?~PqWRCqN%zxU>S{bt_o`N}to
z;clVeTk!ki_dh@VB{&b~$?fxN<;4%In8o(jR&1ZqeHSz*c0qGN^M2~r3E}z?`Ii*+
zUsMoJf6V?Ni~5$mevd=0_h6;h0Thb~JTzwC*#lczLlWDHC@N`-4Q*wQ$jBZz(oxPC
zA(nFXtjM_|2t+=v0Yr92jxt`W%B0>EU+SOjmt0rau%rLBkHRY;q*%|QA(_{SvwZ4+
zg6pCKYiPwpSuN6{ij3HRf{eB4hzuCW&d`miVl8h(2hb68!0u#DL&{#71tLk0+cK@R
zo2iVrxCWWU3kX*c7639<Ph{rat%@`eneMyO%Z><ecNHOzE`|GvtTif=Wco+<%IO-|
z$E}m+Xh4dzKeJa8)m|b>kngE$je;BgkTNbqYYPNJ2}sRZ3f2G#(jfza#1OQH4v?ZP
z7f3P145>ahq;|^XEdTalwcdd`?PlkT$a{v8cm)cGb%YIsO@PQAlR`eCq<`~Xo_ZC;
z356mFsP_2=+0r+7*^NCY$n^=|o5Nlq_7aAfj1fv(`VrrBF!_$2&6UCeNH~Os-Vge2
zZt3))qBpIBukcESG{(zvjKoGTLJQWwHJUTgJfpeb#f{`m>^Y<HiRMc@mz>JMspM5&
zJXCqOd>^6$#fF~b%2BTJRG@gE{4+>LtfQ4rEWPS~v~8xcAISLL_CZ-DCvq+7&wzIQ
zpnSjEk>%=B<Mf45Kk(n0$?rA$$;w)3{heszF^+bW)6@B-s~HtZ=B7zi)Eb#n7v1At
z=2m;Xq!SB<ZUwQeALKv$3{Pgim_nAyai=G`wIofGcsyF<P~>1+;0cZ)L<vuM7go9g
zU{Rm=)I}sAJO)`nZQ?VV%rm_iEa{+7f9(eNiq`=mCbr>kKZo;C=~e5Az>9pg3fe0A
z`@+V)CuVSe7U6k-EJ$TlHfx88Y;@!C7>#cpJuaFxlIa+S;|*A;3&2UhgBQ)foPT>s
z-wGQG@BTN?#7u#$p??iOFVry-UlvZpu6}ds@~L@M%NT^;A(JZ0y?;_mda~K=2t1ts
z?E@$EGOQHeVNSTiJ)Yw{ESVXGZ@?VbGrN6q*cMn&EF<h76cO;|z*T2ctHp7OfIu3;
HX*l}?a5W&e

diff --git a/rsrc/Scraper/proxylister.py b/rsrc/Scraper/proxylister.py
index 9c4ad97..951eac2 100644
--- a/rsrc/Scraper/proxylister.py
+++ b/rsrc/Scraper/proxylister.py
@@ -9,7 +9,7 @@ class Scraper:
         self.config = config
 
     def get_proxies(self):
-        fetch_results = self.client.get( "https://" + self.config.proxylist_url )
+        fetch_results = self.client.get( self.config.proxylist_url )
         proxy_list = self.Parser.scrape( "proxy_list", fetch_results.content )
         return proxy_list
 
@@ -24,7 +24,9 @@ class Scraper:
         @staticmethod
         def proxy_list( text ):
             proxyTable = html.fromstring( text )
-            proxyTable_xpath = proxyTable.xpath( '//table[@class="proxies"]/tbody/tr/@data-domain' )
+            proxyTable_xpath = proxyTable.xpath('//body[@id="mainPage"]/div[@class="container"]/div[@id="content"]/table[@id="searchResult"]/tr/td[@class="site"]/a/text()')
+            for proxy in proxyTable_xpath:
+                print("Available Proxy: {0}".format( proxy ) )
             return proxyTable_xpath
 
     class SessionError( Exception ):
diff --git a/rsrc/Scraper/searcher.py b/rsrc/Scraper/searcher.py
index 08363e8..257391d 100644
--- a/rsrc/Scraper/searcher.py
+++ b/rsrc/Scraper/searcher.py
@@ -3,6 +3,7 @@ from lxml import html
 import urllib
 import re
 import json
+import sys
 
 class Result:
     def __init__(self, title, seeders, leechers, size, author, url):
@@ -32,24 +33,30 @@ class Scraper:
 
     def craft_url(self, protocol, proxy, search_terms):
         # https://pirate.blue/s/?q=Raising+Arizona&category=0&page=0&orderby=99
-        f = { 'q': search_terms, 'category': 0, 'page': 0, 'orderby': 99 }
-        url = str.format( "{0}://{1}/s/?{2}", protocol, proxy, urllib.parse.urlencode(f) )
+        f = { 'q': search_terms, 'page': 0, 'orderby': 99 }
+        # https://thepiratebay0.org/s/?page=0&orderby=0&q=inuyasha
+        # https://thepiratebay0.org/s/?page=0&orderby=0&q=inuyasha
+        url = "{0}://{1}/s/?{2}".format( protocol, proxy, urllib.parse.urlencode(f) )
         print(url)
         return url
 
     def get_results(self, search_terms):
         url = self.craft_url( "https", self.config.proxy, search_terms )
 
-        fetch_results = self.client.get( url )
+        try:
+            fetch_results = self.client.get( url )
+        except requests.exceptions.RequestException as e:
+            print( e, file=sys.stderr )
         results_list = self.Parser.scrape( "results_list", fetch_results.content )
 
         return results_list
 
 
     def get_magnet(self, url):
-        url = "https://" + self.config.proxy + url
-        fetch_results = self.client.get(url)
-
+        try:
+            fetch_results = self.client.get(url)
+        except requests.exceptions.RequestException as e:
+            print( e, file=sys.stderr )
         magnet = self.Parser.scrape( "magnet_link", fetch_results.content )
 
         return magnet
@@ -71,26 +78,40 @@ class Scraper:
             results_buffer = list()
 
             for tr in resultsTable_xpath:
-                title = tr.xpath('td[2]/div[1]/a[1]/text()')
-                seeders = tr.xpath('td[3]/text()')[0]
-                leechers = tr.xpath('td[4]/text()')[0]
-                author = tr.xpath('td[2]/font/a/text()')
-                size_unprocessed = tr.xpath('td[2]/font/text()')[0]
-                url = tr.xpath('td/div[@class="detName"]/a[@class="detLink"]/@href')[0]
+                title = Scraper.Parser.scrape_helper( tr, 'td[2]/div[1]/a[1]/text()' )
+                seeders = Scraper.Parser.scrape_helper( tr, 'td[3]/text()' )
 
+                leechers = Scraper.Parser.scrape_helper( tr, 'td[4]/text()' )
+                url = Scraper.Parser.scrape_helper( tr, 'td/div[@class="detName"]/a[@class="detLink"]/@href' )
+
+                size_unprocessed = Scraper.Parser.scrape_helper( tr, 'td[2]/font/text()' )
 
                 m = re.search('Size (.+?),', size_unprocessed)
 
                 if m:
                     size = m.group(1)
 
+                author = Scraper.Parser.scrape_helper( tr, 'td[2]/font[@class="detDesc"]/*/text()' )
 
+
+
+                print("Result: {0}".format( Result(title, seeders, leechers, size, author, url) ) )
                 results_buffer.append(
                     Result(title, seeders, leechers, size, author, url)
                 )
 
+            # hack
+            nav = results_buffer.pop()
+
             return results_buffer
 
+        @staticmethod
+        def scrape_helper( tr, xpathq ):
+            try:
+                val = tr.xpath( xpathq )[0]
+            except IndexError:
+                val = "0"
+            return val
 
         @staticmethod
         def magnet_link( text ):