simplify image cache

This commit is contained in:
dries.k
2022-08-04 00:04:00 +02:00
parent 298e684049
commit 54b45fa33c
2 changed files with 19 additions and 59 deletions

View File

@@ -1,7 +1,7 @@
import glob
import logging
import os
import time
import uuid
import requests
@@ -35,40 +35,34 @@ class Cache(object):
def clear(self):
self.cursor.execute("DELETE FROM cache")
self.database.commit()
if not self.cache_dir:
logger.error("Cache directory is empty.")
return
for filename in glob.glob(self.cache_dir + "/*"):
os.remove(filename)
def lookup(self, key: str) -> str:
q = self.cursor.execute(
"SELECT filename, cached_on FROM cache WHERE url=?", (key,)
).fetchone()
if q:
if q := self.cursor.execute(
"SELECT filename FROM cache WHERE url=?", (key,)
).fetchone():
# Cache hit
filename, cached_on = q
return filename if os.path.exists(filename) else ""
return q["filename"] if os.path.exists(q["filename"]) else ""
else:
# Cache miss
return ""
def store(
self, key: str, response: requests.Response, add_time: bool = True
) -> str:
def store(self, key: str, response: requests.Response) -> str:
# Create a file and store the response contents
filename = str(time.time()).replace(".", "") if add_time else ""
if "Content-Disposition" in response.headers.keys():
filename += response.headers["Content-Disposition"]
else:
filename += response.url.split("/")[-1]
filepath = os.path.join(self.cache_dir, uuid.uuid4().hex)
filename = "".join([c for c in filename if c.isalpha() or c.isdigit()]).rstrip()
filename = os.path.join(self.cache_dir, filename)
with open(filename, "wb") as f:
with open(filepath, "wb") as f:
f.write(response.content)
self.cursor.execute(
"INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))",
(key, filename),
(key, filepath),
)
self.database.commit()
return filename
return filepath

View File

@@ -22,43 +22,12 @@ class Downloader(object):
"""
return self.session.get(url)
def get_bytes(self, url: str, cached: bool = True, add_time: bool = True) -> bytes:
"""
Get the content of an http get request, as bytes.
Optionally use the cache.
"""
if cached:
# Retrieve from cache
filename = self.cache.lookup(url)
if filename:
with open(filename, "rb") as f:
return f.read()
try:
response = self.get(url)
except Exception as e:
logger.error(f"get_bytes: downloading {url} failed.: {e}")
return b""
if not response.ok:
return b""
if cached:
# Store in cache
self.cache.store(url, response, add_time=add_time)
return response.content
def get_filename(
self, url: str, retrieve_from_cache: bool = True, add_time: bool = True
) -> str:
def get_filename(self, url: str) -> str:
"""
Get the content of an http get request, as a filename.
"""
if retrieve_from_cache:
filename = self.cache.lookup(url)
if filename:
return filename
if filename := self.cache.lookup(url):
return filename
try:
response = self.get(url)
@@ -66,7 +35,4 @@ class Downloader(object):
logger.error(f"get_filename: downloading {url} failed.: {e}")
return ""
if not response.ok:
return ""
return self.cache.store(url, response, add_time=add_time)
return self.cache.store(url, response) if response.ok else ""