simplify image cache

This commit is contained in:
dries.k
2022-08-04 00:04:00 +02:00
parent f6a0036f42
commit 0928a9793a
2 changed files with 19 additions and 59 deletions

View File

@@ -1,7 +1,7 @@
import glob import glob
import logging import logging
import os import os
import time import uuid
import requests import requests
@@ -35,40 +35,34 @@ class Cache(object):
def clear(self): def clear(self):
self.cursor.execute("DELETE FROM cache") self.cursor.execute("DELETE FROM cache")
self.database.commit() self.database.commit()
if not self.cache_dir:
logger.error("Cache directory is empty.")
return
for filename in glob.glob(self.cache_dir + "/*"): for filename in glob.glob(self.cache_dir + "/*"):
os.remove(filename) os.remove(filename)
def lookup(self, key: str) -> str: def lookup(self, key: str) -> str:
q = self.cursor.execute( if q := self.cursor.execute(
"SELECT filename, cached_on FROM cache WHERE url=?", (key,) "SELECT filename FROM cache WHERE url=?", (key,)
).fetchone() ).fetchone():
if q:
# Cache hit # Cache hit
filename, cached_on = q return q["filename"] if os.path.exists(q["filename"]) else ""
return filename if os.path.exists(filename) else ""
else: else:
# Cache miss # Cache miss
return "" return ""
def store( def store(self, key: str, response: requests.Response) -> str:
self, key: str, response: requests.Response, add_time: bool = True
) -> str:
# Create a file and store the response contents # Create a file and store the response contents
filename = str(time.time()).replace(".", "") if add_time else "" filepath = os.path.join(self.cache_dir, uuid.uuid4().hex)
if "Content-Disposition" in response.headers.keys():
filename += response.headers["Content-Disposition"]
else:
filename += response.url.split("/")[-1]
filename = "".join([c for c in filename if c.isalpha() or c.isdigit()]).rstrip() with open(filepath, "wb") as f:
filename = os.path.join(self.cache_dir, filename)
with open(filename, "wb") as f:
f.write(response.content) f.write(response.content)
self.cursor.execute( self.cursor.execute(
"INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))", "INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))",
(key, filename), (key, filepath),
) )
self.database.commit() self.database.commit()
return filename return filepath

View File

@@ -22,43 +22,12 @@ class Downloader(object):
""" """
return self.session.get(url) return self.session.get(url)
def get_bytes(self, url: str, cached: bool = True, add_time: bool = True) -> bytes: def get_filename(self, url: str) -> str:
"""
Get the content of an http get request, as bytes.
Optionally use the cache.
"""
if cached:
# Retrieve from cache
filename = self.cache.lookup(url)
if filename:
with open(filename, "rb") as f:
return f.read()
try:
response = self.get(url)
except Exception as e:
logger.error(f"get_bytes: downloading {url} failed.: {e}")
return b""
if not response.ok:
return b""
if cached:
# Store in cache
self.cache.store(url, response, add_time=add_time)
return response.content
def get_filename(
self, url: str, retrieve_from_cache: bool = True, add_time: bool = True
) -> str:
""" """
Get the content of an http get request, as a filename. Get the content of an http get request, as a filename.
""" """
if retrieve_from_cache: if filename := self.cache.lookup(url):
filename = self.cache.lookup(url) return filename
if filename:
return filename
try: try:
response = self.get(url) response = self.get(url)
@@ -66,7 +35,4 @@ class Downloader(object):
logger.error(f"get_filename: downloading {url} failed.: {e}") logger.error(f"get_filename: downloading {url} failed.: {e}")
return "" return ""
if not response.ok: return self.cache.store(url, response) if response.ok else ""
return ""
return self.cache.store(url, response, add_time=add_time)