Test

aaa

aa

Bugfixing et masse

bbb

asas

aas

asas

aa

as

qq

as

as

as

as

as

aaa

asa

nn

aa

sda

88

asa

sasa

adsa

a

as

asdas

Final version before tests

Fix1

aa

aa

fx

dx

:)

aa

Hopefully last fix

aaa

aaa

bbb

bb

dd

aa

sdae

aa

bb

aa

kk

a

Fix

fx

aa

deploy

fx

asas

bb

as

fx

fx

aa

aa

mess

kk

A

fix

aa

aa

Viwe fix

a

test

a

aa

FX

conjurer_librarian/conjurer_librarian.py

@@ -1,6 +1,7 @@
 import re
 import logging
 import threading
+import time
 from flask import Flask, jsonify, request
 import netrc
 import json
@@ -30,11 +31,15 @@ class Librarian(object):
         netrc_mod = netrc.netrc(NETRC_FILE)
         authTokens = netrc_mod.authenticators("crossref")
         self.cr = Crossref(mailto=authTokens[0])
-        self.last_search = ""
-        self.limit = 20
+        self.limit = 1000
+        self.fetched = 0
+        self.hit = 0
+        self.total = 0
 
     def search_crossref(self, query, uuid):
-        self.last_search = self.cr.works(query = query, limit = self.limit)
+        search_result_from_cr = self.cr.works(query = query, limit = self.limit)
+        self.total = search_result_from_cr['message']['total-results']
+        self.fetched = len(search_result_from_cr['message']['items'])
         #add to sql with UUID
         #sql row - title, doi, author, uuid, last retrieved, is_available
         with open("results.json", "r+", encoding="UTF-8") as data_file:
@@ -45,20 +50,18 @@ class Librarian(object):
                 file_data = []
             # Join new_data with file_data inside emp_details
             summarized_results = []
-            for item in self.last_search['message']['items']:
+            for item in search_result_from_cr['message']['items']:
                 container = "NO"
                 if "container-title" in item:
                     container = "YES"
-                print(item['title'])
-                print(item['DOI'])
-
-                summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container})
-            result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'],  }
+                summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
+            result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
             file_data.append(result)
             print("Saving")
             data_file.seek(0)
             # convert back to json.
             json.dump(file_data, data_file, indent=4)
+            return result
 
 
     def check_if_exists(self,page_url):
@@ -67,8 +70,6 @@ class Librarian(object):
             raise ValueError("URL must start with 'http:' or 'https:'")
         # trunk-ignore(bandit/B310)
         with urlopen(page_url) as response:
-            info = response.info()
-            print(info.get_content_type())
             data = response.read()
             text = data.decode("utf-8")
             for line in text.splitlines():
@@ -90,9 +91,25 @@ class Librarian(object):
             return page_url
         return None
 
+    def refine_query(self, unrefined_reqult):
+        print("Refine result")
+        refined_result = []
+        for item in unrefined_reqult["summary"]:
+            if item["container"] == "NO" and item["title"]:
+                item_link = "https://sci-hub.se/" + str(item['DOI'])
+                if self.check_if_exists(item_link):
+                    refined_result.append(item)
+                    print ("HIT")
+                    print(item["title"])
+                time.sleep(0.01)
+        print("Existing hits")
+        self.hit = len(refined_result)
+        return refined_result
+
     def answer_query(self, query, uuid):
         #Search sql for UUID
         #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        print("Answer result")
         uuid_found = False
         database = None
         with open("results.json", "r", encoding="UTF-8") as new_file:
@@ -107,11 +124,11 @@ class Librarian(object):
                 if item["UUID"] == uuid:
                     print (item["UUID"])
                     uuid_found = True
-                    self.last_search = item
-            if uuid_found:
-                return self.last_search
-        self.search_crossref(query=query, uuid=uuid)
-        return self.last_search
+                    return self.refine_query(item)
+
+        if not uuid_found:
+            answer = self.search_crossref(query=query, uuid=uuid)
+            return self.refine_query(answer)
 
 #example not exists
 #sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
@@ -144,6 +161,9 @@ def query_database():
     app.logger.info(record["UUID"])
     cl = Librarian()
     answer_data = cl.answer_query(record["query"], record["UUID"])
+    app.logger.info(answer_data)
+    summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
+    answer_data = (summary,answer_data)
     return_data = (
         jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
         200,