gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 13:34:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Search webservice (#2)

Browse Source 
* Deployable script for windows file service

* Deploy!

* Server side work
This commit is contained in:
migatu
2024-04-12 23:21:48 +02:00
committed by GitHub
parent d912d9bc1b
commit 0ac9006665
5 changed files with 152 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+39 -16
View File
@@ -12,9 +12,8 @@ from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import scrape_bot
import requests
from typing import Callable
from uuid import UUID
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
@@ -22,7 +21,7 @@ PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
MAX_CR_RESULTS = 150000
ENCODING = "utf-8"
app = Flask(__name__)
@@ -37,13 +36,15 @@ class Librarian(object):
self.cr = Crossref(mailto=authTokens[0])
self.query = query
self.uuid = str(uuid)
self.limit = 3000
self.limit = MAX_CR_RESULTS
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
self.logger = app.logger
self.live_results = []
self.final_result = []
self.not_in_db = []
self.search_result_from_cr = {}
self.done = False
@@ -62,8 +63,7 @@ class Librarian(object):
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.logger.info(self.fetched)
time.sleep(1)
time.sleep(0.1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
@@ -142,20 +142,20 @@ class Librarian(object):
temp.append([item["DOI"], item])
refined_result = await self.check_if_exists(temp)
temp = []
for item in refined_result:
if item["exists"]:
temp.append(item["data"])
refined_result = temp
self.hit = len(refined_result)
return refined_result
self.final_result.append(item["data"])
else:
self.not_in_db.append(item["data"])
self.hit = len(self.final_result)
return self.final_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}")
uuid_found = False
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
@@ -165,7 +165,7 @@ class Librarian(object):
database = json.load(cr_file)
except JSONDecodeError:
pass
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
if database:
for item in database.keys():
if item in self.uuid:
@@ -174,6 +174,10 @@ class Librarian(object):
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
database.update(answer)
@@ -194,6 +198,10 @@ class Librarian(object):
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
#============================= FLASK INTERNALS===============================
@@ -221,11 +229,25 @@ class BackgroundTaskSearch(threading.Thread):
async def _run(self):
while True:
database = None
ndb_database = None
librarian = librarian_queue.get()
self.app.logger.info("STARTED")
result = await librarian.answer_query()
result = {librarian.uuid: result}
self.app.logger.info("Saving to file")
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
except JSONDecodeError:
pass
if ndb_database:
ndb_database.update(librarian.not_in_db)
else:
ndb_database = librarian.not_in_db
ndb_file.truncate(0)
ndb_file.seek(0)
json.dump(ndb_database, ndb_file)
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
@@ -291,14 +313,15 @@ async def get_partial():
#=======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = app.logger
logger.setLevel(logging.DEBUG)
threads = []
#threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
i = 0
for worker in threads:
try:
conjurer_librarian/run_server.bat
+1
View File
@@ -0,0 +1 @@
c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py
conjurer_librarian/scrape_bot.py
+93
View File
@@ -0,0 +1,93 @@
import json
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from requests import Timeout,ConnectTimeout, ConnectionError
from queue import Queue
import re
import time
import random
import logging
from threading import Thread
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
SCR_FILENAME = "40_chunk.txt"
SCR_ENCODING = "utf-8"
WORK_Q = Queue()
random.seed()
def load_ndb_to_q(logger):
logger.info("Loader started")
while True:
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
for item in ndb_database.keys():
logger.info(item)
url = f"https://sci-hub.se/{item}"
WORK_Q.put([item, url, False])
ndb_file.truncate(0)
except JSONDecodeError:
time.sleep(60*60*2)
pass
time.sleep(180)
def check_if_exists_brute_force(logger):
while True:
logger.info("Scraper tick")
item = WORK_Q.get()
page_url =item[1]
logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
blocked = True
try:
# trunk-ignore(bandit/B310)
with urlopen(page_url) as response:
data = response.read()
text = data.decode("utf-8")
if item[2]:
logger.info("Already found")
time.sleep(180)
blocked = False
else:
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
blocked = False
logger.info("Not found")
item[2] = False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
blocked = False
direct_download_link = "https:" + str(m.group(1))
logger.info (direct_download_link)
with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
operated_file.write("\n")
operated_file.write(item[0])
item[2] = True
except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
pass
if blocked:
logger.info(item)
logger.info(text)
logger.error("Got blocked. Fuck.")
time.sleep(60*60*72)
# trunk-ignore(bandit/B311)
rand = random.randint(1,100)
logger.info(f"Sleeping for {3*rand} minutes")
time.sleep(180*rand)
def scraper(logger = None):
if not logger:
logger = logging.getLogger()
logger.setLevel("DEBUG")
h1 = logging.StreamHandler()
logger.addHandler(h1)
logger.info("No logger set. Test run")
loader = Thread(target = load_ndb_to_q, args = (logger,))
scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
loader.start()
scraper.start()
loader.join()
scraper.join()
if __name__ == "__main__":
scraper()
conjurer_librarian/search_bot.py
+19 -15
View File
@@ -5,22 +5,26 @@ from threading import Thread
q = Queue()
#TODO: DATA FOR TEST ONLY
MAXTHREADS = 5
DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#MAXTHREADS = 5
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#TODO: DEPLOYMENT DATA
#MAXTHREADS = 40
#DATABASE_PATH = r'C:\\Database\\chunks\\'
MAXTHREADS = 41
DATABASE_PATH = r'C:\\Database\\chunks\\'
ENCODING = "utf-8"
CHUNk = "_chunk.txt"
_sentinel = object()
result_list = []
WORK_Q_SIZE = 15500000
def producer(out_q, control_q, filename):
def producer(out_q, control_q, filename, logger):
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
lines = operated_file.readlines()
for line in lines:
logger.info(f"Worker {filename} ")
while True:
line = operated_file.readline()
if not line:
break
out_q.put(line)
try:
check = control_q.get(block=False)
@@ -28,11 +32,11 @@ def producer(out_q, control_q, filename):
check = False
pass
if check is _sentinel:
print("TERM signal received")
logger.info("TERM signal received")
control_q.put(check)
break
print(f"Worker finished: {filename}")
out_q.put(_sentinel)
logger.info(f"Worker finished: {filename}")
out_q.put(_sentinel)
#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
@@ -48,7 +52,7 @@ def consumer(in_q, control_q, doi, live_results, logger):
sentinels += 1
else:
for item in result_list:
if item["DOI"] in data:
if item["DOI"] in data and not item["exists"]:
logger.info(data)
logger.info("HIT")
item["exists"] = True
@@ -63,18 +67,18 @@ def consumer(in_q, control_q, doi, live_results, logger):
def search_for_doi(doi, live_results, logger):
threads = []
work_q = Queue(maxsize=200000)
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0,MAXTHREADS):
#TODO: TEST DATA
filename = "test" + str(i) + CHUNk
#filename = "test" + str(i) + CHUNk
#TODO: DEPLOYMENT DATA
#filename = str(i) + CHUNk
filename = str(i) + CHUNk
logger.info(f"Creating worker thread no: {i}")
threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
for worker in threads:
worker.start()
for worker in threads: