simplify image cache
This commit is contained in:
@@ -1,7 +1,7 @@
|
|||||||
import glob
|
import glob
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import time
|
import uuid
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
@@ -35,40 +35,34 @@ class Cache(object):
|
|||||||
def clear(self):
|
def clear(self):
|
||||||
self.cursor.execute("DELETE FROM cache")
|
self.cursor.execute("DELETE FROM cache")
|
||||||
self.database.commit()
|
self.database.commit()
|
||||||
|
|
||||||
|
if not self.cache_dir:
|
||||||
|
logger.error("Cache directory is empty.")
|
||||||
|
return
|
||||||
|
|
||||||
for filename in glob.glob(self.cache_dir + "/*"):
|
for filename in glob.glob(self.cache_dir + "/*"):
|
||||||
os.remove(filename)
|
os.remove(filename)
|
||||||
|
|
||||||
def lookup(self, key: str) -> str:
|
def lookup(self, key: str) -> str:
|
||||||
q = self.cursor.execute(
|
if q := self.cursor.execute(
|
||||||
"SELECT filename, cached_on FROM cache WHERE url=?", (key,)
|
"SELECT filename FROM cache WHERE url=?", (key,)
|
||||||
).fetchone()
|
).fetchone():
|
||||||
if q:
|
|
||||||
# Cache hit
|
# Cache hit
|
||||||
filename, cached_on = q
|
return q["filename"] if os.path.exists(q["filename"]) else ""
|
||||||
return filename if os.path.exists(filename) else ""
|
|
||||||
else:
|
else:
|
||||||
# Cache miss
|
# Cache miss
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
def store(
|
def store(self, key: str, response: requests.Response) -> str:
|
||||||
self, key: str, response: requests.Response, add_time: bool = True
|
|
||||||
) -> str:
|
|
||||||
# Create a file and store the response contents
|
# Create a file and store the response contents
|
||||||
filename = str(time.time()).replace(".", "") if add_time else ""
|
filepath = os.path.join(self.cache_dir, uuid.uuid4().hex)
|
||||||
if "Content-Disposition" in response.headers.keys():
|
|
||||||
filename += response.headers["Content-Disposition"]
|
|
||||||
else:
|
|
||||||
filename += response.url.split("/")[-1]
|
|
||||||
|
|
||||||
filename = "".join([c for c in filename if c.isalpha() or c.isdigit()]).rstrip()
|
with open(filepath, "wb") as f:
|
||||||
filename = os.path.join(self.cache_dir, filename)
|
|
||||||
|
|
||||||
with open(filename, "wb") as f:
|
|
||||||
f.write(response.content)
|
f.write(response.content)
|
||||||
|
|
||||||
self.cursor.execute(
|
self.cursor.execute(
|
||||||
"INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))",
|
"INSERT INTO cache (url, filename, cached_on) VALUES(?, ?, datetime('now', 'localtime'))",
|
||||||
(key, filename),
|
(key, filepath),
|
||||||
)
|
)
|
||||||
self.database.commit()
|
self.database.commit()
|
||||||
return filename
|
return filepath
|
||||||
|
|||||||
@@ -22,42 +22,11 @@ class Downloader(object):
|
|||||||
"""
|
"""
|
||||||
return self.session.get(url)
|
return self.session.get(url)
|
||||||
|
|
||||||
def get_bytes(self, url: str, cached: bool = True, add_time: bool = True) -> bytes:
|
def get_filename(self, url: str) -> str:
|
||||||
"""
|
|
||||||
Get the content of an http get request, as bytes.
|
|
||||||
Optionally use the cache.
|
|
||||||
"""
|
|
||||||
if cached:
|
|
||||||
# Retrieve from cache
|
|
||||||
filename = self.cache.lookup(url)
|
|
||||||
if filename:
|
|
||||||
with open(filename, "rb") as f:
|
|
||||||
return f.read()
|
|
||||||
|
|
||||||
try:
|
|
||||||
response = self.get(url)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"get_bytes: downloading {url} failed.: {e}")
|
|
||||||
return b""
|
|
||||||
|
|
||||||
if not response.ok:
|
|
||||||
return b""
|
|
||||||
|
|
||||||
if cached:
|
|
||||||
# Store in cache
|
|
||||||
self.cache.store(url, response, add_time=add_time)
|
|
||||||
|
|
||||||
return response.content
|
|
||||||
|
|
||||||
def get_filename(
|
|
||||||
self, url: str, retrieve_from_cache: bool = True, add_time: bool = True
|
|
||||||
) -> str:
|
|
||||||
"""
|
"""
|
||||||
Get the content of an http get request, as a filename.
|
Get the content of an http get request, as a filename.
|
||||||
"""
|
"""
|
||||||
if retrieve_from_cache:
|
if filename := self.cache.lookup(url):
|
||||||
filename = self.cache.lookup(url)
|
|
||||||
if filename:
|
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -66,7 +35,4 @@ class Downloader(object):
|
|||||||
logger.error(f"get_filename: downloading {url} failed.: {e}")
|
logger.error(f"get_filename: downloading {url} failed.: {e}")
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
if not response.ok:
|
return self.cache.store(url, response) if response.ok else ""
|
||||||
return ""
|
|
||||||
|
|
||||||
return self.cache.store(url, response, add_time=add_time)
|
|
||||||
|
|||||||
Reference in New Issue
Block a user