get_flibusta.py

   1 #!/usr/bin/python
   2
   3 from bs4 import BeautifulSoup
   4 import urllib3
   5 import db
   6 import re
   7 import os
   8
   9 pattern = re.compile("^f.fb2.*")
  10 os.environ['http_proxy']='http://192.168.1.100:3128'
  11 os.environ['no_proxy']='localhost,127.0.0.1'
  12 proxies = {'http': 'http://localhost:3128'}
  13
  14
  15 for host in ['flibustahezeous3.onion','flibusta.i2p']:
  16
  17   matched = False
  18   try:
  19     print("Trying %s" % (host))
  20     http = urllib3.PoolManager()
  21     resp = http.request('GET', "http://%s/daily/" % (host))
  22     if resp.status == 200:
  23       html_page = resp.data.decode('utf-8')
  24       html = BeautifulSoup(html_page,"html.parser")
  25
  26       os_command = "wget -c -q -P \"%s\" http://%s/daily/%s" % (db.upload_files,host,'%s')
  27
  28       for link in html.findAll('a'):
  29         file = link.get("href")
  30         print(file)
  31         if pattern.match(file):
  32           print("Pattern matched")
  33           matched = True
  34           if not db.TestArchive(file):
  35             print("Processing %s" % file)
  36             if os.system(os_command % file) == 0:
  37               db.MarkArchive(file)
  38               db.Commit()
  39   except:
  40     raise
  41     matched = False
  42
  43   if matched:
  44     break
  45
  46 if matched:
  47   print("Got from %s" % host)
  48 else:
  49   print("Failed to get")