simplify image cache
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import requests
|
||||
|
||||
@@ -35,40 +35,34 @@ class Cache(object):
|
||||
def clear(self):
|
||||
self.cursor.execute("DELETE FROM cache")
|
||||
self.database.commit()
|
||||
|
||||
if not self.cache_dir:
|
||||
logger.error("Cache directory is empty.")
|
||||
return
|
||||
|
||||
for filename in glob.glob(self.cache_dir + "/*"):
|
||||
os.remove(filename)
|
||||
|
||||
def lookup(self, key: str) -> str:
|
||||
q = self.cursor.execute(
|
||||
"SELECT filename, cached_on FROM cache WHERE url=?", (key,)
|
||||
).fetchone()
|
||||
if q:
|
||||
if q := self.cursor.execute(
|
||||
"SELECT filename FROM cache WHERE url=?", (key,)
|
||||
).fetchone():
|
||||
# Cache hit
|
||||
filename, cached_on = q
|
||||
return filename if os.path.exists(filename) else ""
|
||||
return q["filename"] if os.path.exists(q["filename"]) else ""
|
||||
else:
|
||||
# Cache miss
|
||||
return ""
|
||||
|
||||
def store(
|
||||
self, key: str, response: requests.Response, add_time: bool = True
|
||||
) -> str:
|
||||
def store(self, key: str, response: requests.Response) -> str:
|
||||
# Create a file and store the response contents
|
||||
filename = str(time.time()).replace(".", "") if add_time else ""
|
||||
if "Content-Disposition" in response.headers.keys():
|
||||
filename += response.headers["Content-Disposition"]
|
||||
else:
|
||||
filename += response.url.split("/")[-1]
|
||||
filepath = os.path.join(self.cache_dir, uuid.uuid4().hex)
|
||||
|
||||
filename = "".join([c for c in filename if c.isalpha() or c.isdigit()]).rstrip()
|
||||
filename = os.path.join(self.cache_dir, filename)
|
||||
|
||||
with open(filename, "wb") as f:
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
self.cursor.execute(
|
||||
"INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))",
|
||||
(key, filename),
|
||||
(key, filepath),
|
||||
)
|
||||
self.database.commit()
|
||||
return filename
|
||||
return filepath
|
||||
|
||||
@@ -22,43 +22,12 @@ class Downloader(object):
|
||||
"""
|
||||
return self.session.get(url)
|
||||
|
||||
def get_bytes(self, url: str, cached: bool = True, add_time: bool = True) -> bytes:
|
||||
"""
|
||||
Get the content of an http get request, as bytes.
|
||||
Optionally use the cache.
|
||||
"""
|
||||
if cached:
|
||||
# Retrieve from cache
|
||||
filename = self.cache.lookup(url)
|
||||
if filename:
|
||||
with open(filename, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
try:
|
||||
response = self.get(url)
|
||||
except Exception as e:
|
||||
logger.error(f"get_bytes: downloading {url} failed.: {e}")
|
||||
return b""
|
||||
|
||||
if not response.ok:
|
||||
return b""
|
||||
|
||||
if cached:
|
||||
# Store in cache
|
||||
self.cache.store(url, response, add_time=add_time)
|
||||
|
||||
return response.content
|
||||
|
||||
def get_filename(
|
||||
self, url: str, retrieve_from_cache: bool = True, add_time: bool = True
|
||||
) -> str:
|
||||
def get_filename(self, url: str) -> str:
|
||||
"""
|
||||
Get the content of an http get request, as a filename.
|
||||
"""
|
||||
if retrieve_from_cache:
|
||||
filename = self.cache.lookup(url)
|
||||
if filename:
|
||||
return filename
|
||||
if filename := self.cache.lookup(url):
|
||||
return filename
|
||||
|
||||
try:
|
||||
response = self.get(url)
|
||||
@@ -66,7 +35,4 @@ class Downloader(object):
|
||||
logger.error(f"get_filename: downloading {url} failed.: {e}")
|
||||
return ""
|
||||
|
||||
if not response.ok:
|
||||
return ""
|
||||
|
||||
return self.cache.store(url, response, add_time=add_time)
|
||||
return self.cache.store(url, response) if response.ok else ""
|
||||
|
||||
Reference in New Issue
Block a user