New deasrc

conjurer_librarian/conjurer_librarian.py

@@ -26,6 +26,7 @@ from queue import Queue
 import requests
 import scrape_bot
 import search_bot
+#import search_bot2 as search_bot
 from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
@@ -37,7 +38,10 @@ PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
 BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 7000
+
+MAX_CR_RESULTS = 1000
+#TEST PURPOSES ONLY!
+#MAX_CR_RESULTS = 5
 
 ENCODING = "utf-8"
 
@@ -114,14 +118,15 @@ class Librarian(object):
         self.app.logger.info("STARTED SEARCH")
         
         if not deep_search:
-            cr_result = self.cr.works(query=query, limit=1000)
+            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
+            cr_result = self.cr.works(query=query, limit=query_limit)
             self.search_result_from_cr.update(cr_result)
             self.total = cr_result["message"]["total-results"]
             self.fetched += len(cr_result["message"]["items"])
             self.app.logger.info(self.total)
             self.app.logger.info(self.fetched)
             while self.total > self.fetched and self.limit > self.fetched:
-                tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
+                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
                 cr_result["message"]["items"].extend(tmp_result["message"]["items"])
                 self.total = tmp_result["message"]["total-results"]
                 self.fetched = len(cr_result["message"]["items"])
@@ -130,7 +135,7 @@ class Librarian(object):
                 time.sleep(0.1)
 
         else:
-            cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
+            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
             result = cr_result[0]
             for item in cr_result[1:]:
                 result["message"]["items"].extend(item["message"]["items"])