import requests
from lxml import html
class Scraper:
def __init__( self, config ):
# the request client, belongs to session even if no "user session" is needed
self.client = requests.Session()
self.config = config
def get_proxies(self):
fetch_results = self.client.get( "https://" + self.config.proxylist_url )
proxy_list = self.Parser.scrape( "proxy_list", fetch_results.content )
return proxy_list
class Parser:
@staticmethod
def scrape( datapoint, text ):
cases = {
"proxy_list": Scraper.Parser.proxy_list
}
return cases[ datapoint ]( text )
@staticmethod
def proxy_list( text ):
proxyTable = html.fromstring( text )
proxyTable_xpath = proxyTable.xpath( '//table[@class="proxies"]/tbody/tr/@data-domain' )
return proxyTable_xpath
class SessionError( Exception ):
def __init__( self, value ):
self.value = value