summaryrefslogtreecommitdiffstats
path: root/gather.py
diff options
context:
space:
mode:
authorAnton Luka Šijanec <anton@sijanec.eu>2022-11-14 19:34:37 +0100
committerAnton Luka Šijanec <anton@sijanec.eu>2022-11-14 19:34:37 +0100
commit323071339447395ddb783b52603ab6534cc16a2e (patch)
tree9a5a7dea79a72cf28715d20fbf7f96e4be0c1ca2 /gather.py
parentsometimes, wrong isbns are entered on the server 1170903 (diff)
downloadbiblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar.gz
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar.bz2
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar.lz
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar.xz
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.tar.zst
biblos-stat-323071339447395ddb783b52603ab6534cc16a2e.zip
Diffstat (limited to 'gather.py')
-rwxr-xr-xgather.py25
1 files changed, 15 insertions, 10 deletions
diff --git a/gather.py b/gather.py
index 5950363..64146b2 100755
--- a/gather.py
+++ b/gather.py
@@ -4,6 +4,7 @@ import logging
from time import localtime, mktime, time
import requests
from base64 import b64decode
+from datetime import datetime, timedelta, timezone
try:
from sqlalchemy import Table, MetaData, Integer, BigInteger, String, Column, Table, ForeignKey, create_engine, select
from sqlalchemy.orm import declarative_base, relationship, Session
@@ -58,16 +59,14 @@ starting_acsm_id = 177238
guaranteed_large_acsm_id = 1170487
logger.debug(f"created metadata.")
-force_acsm_id = 0
-valid_acsms = 0
-only_isbn_acsms = 0
-failed_acsms = 0
-failed_acsms_not200 = 0
-failed_acsms_not200_in_a_row = 0
-hmfan2iarts = 100 # how many failed acsms not 200 in a row to stop
-
-try:
+def update(hmfan2iarts=100):
+ force_acsm_id = 0
+ valid_acsms = 0
+ only_isbn_acsms = 0
+ failed_acsms = 0
+ failed_acsms_not200 = 0
+ failed_acsms_not200_in_a_row = 0
with Session(engine) as session:
while True:
if force_acsm_id != 0:
@@ -86,6 +85,9 @@ try:
if (r.status_code == 200):
failed_acsms_not200_in_a_row = 0
if r.status_code != 200:
+ if datetime.strptime(borrow.purchase, "%Y-%m-%dT%H:%M:%S%z") > datetime.now(timezone.utc) - timedelta(hours=1):
+ logger.info(f"we are done for now, as server responded with {r.status_code} for queried acsm id {acsm_id} and the last requested acsm was created less than an hour ago")
+ break
logger.warning(f"received http response with error code not 200 (it is {r.status_code}). if this continues for {hmfan2iarts-failed_acsms_not200_in_a_row} more requests, I'll assume there are no more borrows on the server.")
failed_acsms_not200 += 1
failed_acsms_not200_in_a_row += 1
@@ -177,7 +179,10 @@ try:
session.add(borrow)
session.commit()
valid_acsms += 1
+ logger.info(f"In this update, {r['valid_acsms']} valid acsms were stored, {r['only_isbn_acsms']} acsms had only isbn and no other data available and {r['failed_acsms']} acsms failed to be received with response code 200 and {r['failed_acsms_not200']} acsms failed to be received but did not return 200. Last valid requested acsm was {r['acsm_id']}. Thank you for cooperation.")
+ return {"valid_acsms": valid_acsms, "only_isbn_acsms": only_isbn_acsms, "failed_acsms": failed_acsms, "failed_acsms_not200": failed_acsms_not200, "acsm_id": acsm_id}
+try:
+ r = update()
except KeyboardInterrupt:
logger.warning(f"Keyboard interrupt. Exiting. I hope this terminated cleanly. Last requested acsm was discarded.")
-logger.info(f"In this session, {valid_acsms} valid acsms were stored, {only_isbn_acsms} acsms had only isbn and no other data available and {failed_acsms} acsms failed to be received with response code 200 and {failed_acsms_not200} acsms failed to be received but did not return 200. Last valid requested acsm was {acsm_id}. Thank you for cooperation.")