Добавлена процедура поиска и удаления дубликатов.

[openlib.git] / get_flibusta.py

diff --git a/get_flibusta.py b/get_flibusta.py
index 654b5bed8555ccb5eb7b9b14714887dd0505de97..13768c47930a9be4b4477d29c74460088a45f8da 100755 (executable)
--- a/get_flibusta.py
+++ b/get_flibusta.py
@@ -7,19 +7,19 @@ import re
 import os
 
 pattern = re.compile("^f.fb2.*")
 import os
 
 pattern = re.compile("^f.fb2.*")

-os.environ['http_proxy']='http://localhost:3128'
+os.environ['http_proxy']='http://192.168.1.100:3128'

 os.environ['no_proxy']='localhost,127.0.0.1'
 proxies = {'http': 'http://localhost:3128'}
 
 
 os.environ['no_proxy']='localhost,127.0.0.1'
 proxies = {'http': 'http://localhost:3128'}
 
 

-for host in ['flibusta.net','flibustahezeous3.onion','flibusta.i2p']:
+for host in ['flibustahezeous3.onion','flibusta.i2p','flibusta.net']:

 
   try:
     print "Trying %s" % (host)
     html_page = urllib.urlopen("http://%s/daily/" % (host))
     html = BeautifulSoup(html_page)
 
 
   try:
     print "Trying %s" % (host)
     html_page = urllib.urlopen("http://%s/daily/" % (host))
     html = BeautifulSoup(html_page)
 

-    os_command = "wget -c -P \"%s\" http://%s/daily/%s" % (db.upload_files,host,'%s')
+    os_command = "wget -c -q -P \"%s\" http://%s/daily/%s" % (db.upload_files,host,'%s')

     print os_command   
     matched = False 
 
     print os_command   
     matched = False