improve artwork resolution with parallel requests and fallbacks

aanderse · aanderse · commit 7d021a11d5f6 · 2025-12-25T12:21:42.000-05:00
- Use ThreadPoolExecutor to check artwork URLs in parallel (20 workers)
- Fall back to header.jpg when poster/hero images unavailable
- Cache HEAD request results for 2 months
- Reduces load time from ~25s to ~2s (first load) or ~0.1s (cached)
diff --git a/resources/arts.py b/resources/arts.py
@@ -1,111 +1,195 @@
-import xbmcaddon
-import xbmcvfs
-
 import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import timedelta
+
 import requests
 import requests_cache
+import xbmcaddon
+import xbmcvfs
 
 from .util import log
 
 __addon__ = xbmcaddon.Addon()
-artFallbackEnabled = __addon__.getSetting("enable-art-fallback") == 'true'  # Kodi stores boolean settings as strings
-monthsBeforeArtsExpiration = int(__addon__.getSetting("arts-expire-after-months"))  # Default is 2 months
 
-# define the cache file to reside in the ..\Kodi\userdata\addon_data\(your addon)
+# Cache setup for HEAD requests
 addonUserDataFolder = xbmcvfs.translatePath(__addon__.getAddonInfo('profile'))
-ART_AVAILABILITY_CACHE_FILE = xbmcvfs.translatePath(os.path.join(addonUserDataFolder, 'requests_cache_arts'))
-
-cached_requests = requests_cache.core.CachedSession(ART_AVAILABILITY_CACHE_FILE, backend='sqlite',
-                                                    expire_after= timedelta(weeks=4*monthsBeforeArtsExpiration),
-                                                    allowable_methods=('HEAD',), allowable_codes=(200, 404),
-                                                    old_data_on_error=True,
-                                                    fast_save=True)
-# Existing Steam art types urls, to format to format with appid / img_icon_path
-STEAM_ARTS_TYPES = {  # img_icon_path is provided by steam API to get the icon. https://developer.valvesoftware.com/wiki/Steam_Web_API#GetOwnedGames_.28v0001.29
-    'poster': 'http://cdn.akamai.steamstatic.com/steam/apps/{appid}/library_600x900.jpg',  # Can return 404
-    'hero': 'http://cdn.akamai.steamstatic.com/steam/apps/{appid}/library_hero.jpg',  # Can return 404
-    'header': 'http://cdn.akamai.steamstatic.com/steam/apps/{appid}/header.jpg',
-    'generated_bg': 'http://cdn.akamai.steamstatic.com/steam/apps/{appid}/page_bg_generated_v6b.jpg',  # Auto generated background with a shade of blue.
+ART_CACHE_FILE = xbmcvfs.translatePath(os.path.join(addonUserDataFolder, 'requests_cache_arts'))
+
+# Cache HEAD request results for 2 months
+cached_session = requests_cache.CachedSession(
+    ART_CACHE_FILE,
+    backend='sqlite',
+    expire_after=timedelta(weeks=8),
+    allowable_methods=('HEAD',),
+    allowable_codes=(200, 404),
+    old_data_on_error=True
+)
+
+# Steam art URL templates
+STEAM_ARTS = {
+    'poster': 'https://cdn.akamai.steamstatic.com/steam/apps/{appid}/library_600x900.jpg',
+    'hero': 'https://cdn.akamai.steamstatic.com/steam/apps/{appid}/library_hero.jpg',
+    'header': 'https://cdn.akamai.steamstatic.com/steam/apps/{appid}/header.jpg',
+    'logo': 'https://cdn.akamai.steamstatic.com/steam/apps/{appid}/logo.png',
     'icon': 'https://steamcdn-a.akamaihd.net/steamcommunity/public/images/apps/{appid}/{img_icon_path}.jpg',
-    'clearlogo': 'http://cdn.akamai.steamstatic.com/steam/apps/{appid}/logo.png'  # Can return 404
 }
 
-# Dictionary containing for each art type, a url for the art (to format with appid / img_icon_path afterwards), and a fallback art type.
-# Having no fallback also means that the art url won't be tested
-ARTS_ASSIGNMENTS = {
-    'poster': {'url': STEAM_ARTS_TYPES['poster'], 'fallback': 'landscape'},
-    'banner': {'url': STEAM_ARTS_TYPES['hero'], 'fallback': 'landscape'},
-    'fanart': {'url': STEAM_ARTS_TYPES['hero'], 'fallback': 'fanart1'},
-    'fanart1': {'url': STEAM_ARTS_TYPES['header'], 'fallback': None},
-    'fanart2': {'url': STEAM_ARTS_TYPES['generated_bg'], 'fallback': None},  # Multiple fanart https://kodi.wiki/view/Artwork_types#fanart.23
-    'landscape': {'url': STEAM_ARTS_TYPES['header'], 'fallback': None},
-    'thumb': {'url': STEAM_ARTS_TYPES['header'], 'fallback': None},
-    'icon': {'url': STEAM_ARTS_TYPES['icon'], 'fallback': None},
-    'clearlogo': {'url': STEAM_ARTS_TYPES['clearlogo'], 'fallback': None}
+# Map Kodi art types to Steam art (primary -> fallback)
+# header.jpg is available for all games, so it's the ultimate fallback
+ART_TYPE_CONFIG = {
+    'poster': {'primary': 'poster', 'fallback': 'header'},
+    'banner': {'primary': 'hero', 'fallback': 'header'},
+    'fanart': {'primary': 'hero', 'fallback': 'header'},
+    'fanart1': {'primary': 'header', 'fallback': None},
+    'fanart2': {'primary': 'header', 'fallback': None},
+    'landscape': {'primary': 'header', 'fallback': None},
+    'thumb': {'primary': 'header', 'fallback': None},
+    'icon': {'primary': 'icon', 'fallback': None},
+    'clearlogo': {'primary': 'logo', 'fallback': None},
 }
 
 
-def is_art_url_available(url, timeout=2):
+def check_url_exists(url, timeout=2):
     """
-    Sends a HEAD request to check if an online resource is available. Uses a cache mechanism to speed things up or serve offline if a connection is unavailable.
-
-    :param url: url to check availability
-    :param timeout: timeout of the request in seconds. Default is 2
-    :return: boolean False if the status code is between 400&600 , True otherwise
+    Check if a URL exists (returns 200). Results are cached.
     """
-    result = False
     try:
-        response = cached_requests.head(url, timeout=timeout)
-        if not 400 <= response.status_code < 600:  # We consider valid any status codes below 400 or above 600
-            result = True
-    except IOError:
-        result = False
-    return result
+        response = cached_session.head(url, timeout=timeout)
+        return response.status_code == 200
+    except:
+        return False
+
+
+def get_art_url(art_key, appid, img_icon_path=''):
+    """Get a formatted Steam art URL."""
+    template = STEAM_ARTS.get(art_key)
+    if not template:
+        return None
+    return template.format(appid=appid, img_icon_path=img_icon_path)
 
 
-def resolve_art_url(art_type, appid, img_icon_path='', art_fallback_enabled=artFallbackEnabled):
+def resolve_art_for_game(appid, img_icon_path=''):
     """
-    Resolve the art url of a specified game/app, for a given art type defined in the :const:`ARTS_DATA` dictionary.
-    Handles fallback to another art type if needed (ie the requested one is unavailable and fallback is enabled).
-
-    :param art_type: a valid art type, defined in :const:`ARTS_DATA`
-    :param appid: appid of the game/app we want to get the art for.
-    :param img_icon_path: A path provided by steam to get the icon art url. https://developer.valvesoftware.com/wiki/Steam_Web_API#GetOwnedGames_.28v0001.29
-    :param art_fallback_enabled: Whether to fall back to another art type if an art is unavailable. Defaults to the user addon settings, which default to true
-    :return: resolved art URL. Can be the URL of another available art if .
+    Resolve all art URLs for a single game, checking availability and using fallbacks.
+    Returns a dict of {art_type: url}.
     """
-    valid_art_url = None
-    requested_art = ARTS_ASSIGNMENTS.get(art_type, None)
+    art_dict = {}
+    urls_to_check = []
 
-    while valid_art_url is None and requested_art is not None:  # If the current media type is defined and we did not find a valid url yet
-        art_url = requested_art.get('url').format(appid=appid, img_icon_path=img_icon_path)  # We replace "{appid}" and "{img_icon_path}" in the url
-        fallback_art_type = requested_art.get("fallback", None)
-        if (not art_fallback_enabled) or (fallback_art_type is None) or is_art_url_available(art_url):
-            # If art fallback is disabled, or if there is no fallback defined, we directly assume the art url as valid.
-            # Otherwise, if art fallback is enabled and there is a fallback defined, we check if is_art_url_available before proceeding
-            valid_art_url = art_url
-        else:  # If art fallback is enabled and art is not available, we set the current art data to the defined fallback, before retrying.
-            requested_art = ARTS_ASSIGNMENTS.get(fallback_art_type, None)  # Art data will be None if the fallback_art_type does not exist in the art_urls dict
+    # Build list of primary URLs that need checking (have fallbacks)
+    for art_type, config in ART_TYPE_CONFIG.items():
+        primary_url = get_art_url(config['primary'], appid, img_icon_path)
+        fallback_key = config.get('fallback')
 
-    if valid_art_url is None:  # If the previous loop could not find a valid media url among the defined art types
-        log("Issue resolving media {0} for app id {1}".format(art_type, appid))
+        if fallback_key:
+            # This art type has a fallback, so we need to check if primary exists
+            urls_to_check.append((art_type, primary_url, fallback_key))
+        else:
+            # No fallback, just use the primary URL directly
+            art_dict[art_type] = primary_url
 
-    return valid_art_url
+    # Check all primary URLs that have fallbacks
+    for art_type, primary_url, fallback_key in urls_to_check:
+        if check_url_exists(primary_url):
+            art_dict[art_type] = primary_url
+        else:
+            art_dict[art_type] = get_art_url(fallback_key, appid, img_icon_path)
 
+    return art_dict
 
-def delete_cache():
+
+def resolve_art_for_all_games(games):
+    """
+    Resolve art URLs for all games in parallel.
+
+    :param games: list of game dicts with 'appid' and 'img_icon_url' keys
+    :return: dict mapping appid -> art_dict
+    """
+    start_time = time.time()
+
+    # First, collect all URLs that need checking
+    urls_to_check = set()
+    game_art_info = []  # (appid, img_icon_path, [(art_type, primary_url, fallback_key), ...])
+
+    for game in games:
+        appid = str(game['appid'])
+        img_icon_path = game.get('img_icon_url', '')
+
+        art_checks = []
+        for art_type, config in ART_TYPE_CONFIG.items():
+            if config.get('fallback'):
+                primary_url = get_art_url(config['primary'], appid, img_icon_path)
+                urls_to_check.add(primary_url)
+                art_checks.append((art_type, primary_url, config['fallback']))
+
+        game_art_info.append((appid, img_icon_path, art_checks))
+
+    # Check all URLs in parallel
+    url_exists = {}
+    check_start = time.time()
+
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        future_to_url = {executor.submit(check_url_exists, url): url for url in urls_to_check}
+        for future in as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                url_exists[url] = future.result()
+            except:
+                url_exists[url] = False
+
+    log("Checked {} URLs in {:.2f}s".format(len(urls_to_check), time.time() - check_start))
+
+    # Build art dictionaries using check results
+    results = {}
+    for appid, img_icon_path, art_checks in game_art_info:
+        art_dict = {}
+
+        # Add art types that don't need checking
+        for art_type, config in ART_TYPE_CONFIG.items():
+            if not config.get('fallback'):
+                art_dict[art_type] = get_art_url(config['primary'], appid, img_icon_path)
+
+        # Add art types that were checked
+        for art_type, primary_url, fallback_key in art_checks:
+            if url_exists.get(primary_url, False):
+                art_dict[art_type] = primary_url
+            else:
+                art_dict[art_type] = get_art_url(fallback_key, appid, img_icon_path)
+
+        results[appid] = art_dict
+
+    log("Resolved art for {} games in {:.2f}s".format(len(games), time.time() - start_time))
+    return results
+
+
+def resolve_art_url(art_type, appid, img_icon_path=''):
     """
-    Deletes the cache containing the data about which art types are available or not
+    Resolve a single art URL (legacy interface).
+    For better performance, use resolve_art_for_all_games() instead.
     """
-    # If Kodi's request-cache module is updated to >0.7.3 , we will only need to issue cached_requests.cache.clear() which will handle all scenarios. Until then, we must recreate the backend ourselves    
+    config = ART_TYPE_CONFIG.get(art_type)
+    if not config:
+        return None
+
+    primary_url = get_art_url(config['primary'], appid, img_icon_path)
+    fallback_key = config.get('fallback')
+
+    if fallback_key and not check_url_exists(primary_url):
+        return get_art_url(fallback_key, appid, img_icon_path)
+
+    return primary_url
+
+
+def delete_cache():
+    """Delete the art availability cache."""
     try:
-        cached_requests.cache.clear()
-    except Exception:
-        log('Failed to clear cache. Attempting manual deletion')
+        cached_session.cache.clear()
+        log("Art cache cleared successfully")
+    except Exception as e:
+        log("Failed to clear art cache: {}".format(e))
         try:
-            os.remove(ART_AVAILABILITY_CACHE_FILE + ".sqlite")
+            os.remove(ART_CACHE_FILE + ".sqlite")
+            log("Art cache file deleted")
         except:
-            log('Failed to delete cache file')
-        cached_requests.cache.responses = requests_cache.backends.storage.dbdict.DbPickleDict(ART_AVAILABILITY_CACHE_FILE + ".sqlite", 'responses', fast_save=True)
-        cached_requests.cache.keys_map = requests_cache.backends.storage.dbdict.DbDict(ART_AVAILABILITY_CACHE_FILE + ".sqlite", 'urls')
+            log("Failed to delete art cache file")
diff --git a/resources/main.py b/resources/main.py
@@ -1,6 +1,7 @@
 import os
 import routing
 import sys
+import time
 import xbmcplugin
 
 from . import arts
@@ -25,25 +26,34 @@ def index():
 
 @plugin.route('/all')
 def all_games():
+    start_time = time.time()
+
     if not all_required_credentials_available():
         return
 
     try:
+        api_start = time.time()
         steam_games_details = steam.get_user_games(__addon__.getSetting('steam-key'), __addon__.getSetting('steam-id'))
+        log("Steam API call took {:.2f}s".format(time.time() - api_start))
 
     except IOError as e:
         # something went wrong, can't scan the steam library
         show_error(e, 'An unexpected error has occurred while contacting Steam. Please ensure your Steam credentials are correct and then try again. '
                       'If this problem persists please contact support.')
         return
 
+    items_start = time.time()
     directory_items = create_directory_items(steam_games_details)
+    log("Creating directory items took {:.2f}s for {} games".format(time.time() - items_start, len(directory_items)))
+
     xbmcplugin.addDirectoryItems(plugin.handle, directory_items)
 
     xbmcplugin.addSortMethod(plugin.handle, xbmcplugin.SORT_METHOD_LABEL)
     xbmcplugin.addSortMethod(plugin.handle, xbmcplugin.SORT_METHOD_PLAYCOUNT)
     xbmcplugin.endOfDirectory(plugin.handle, succeeded=True)
 
+    log("Total /all route took {:.2f}s".format(time.time() - start_time))
+
 
 @plugin.route('/installed')
 def installed_games():
@@ -138,6 +148,12 @@ def create_directory_items(app_entries):
     xbmcplugin.setContent(plugin.handle, "movies")
     # TODO setContent to games when more skins support this content type.
 
+    # Convert to list to allow multiple iterations and get count
+    app_entries = list(app_entries)
+
+    # Resolve all artwork URLs in parallel (with fallback checking)
+    all_art = arts.resolve_art_for_all_games(app_entries)
+
     directory_items = []
     for app_entry in app_entries:
         appid = str(app_entry['appid'])
@@ -153,33 +169,14 @@ def create_directory_items(app_entries):
                                   ('Install', 'RunPlugin(' + plugin.url_for(install, appid=appid) + ')')],
                                  replaceItems=True)  # Since we set the content type to "movies", default movie context elements may appear. We replace them.
 
-        art_dictionary = create_arts_dictionary(app_entry)
-        item.setArt(art_dictionary)
+        # Use pre-resolved artwork
+        item.setArt(all_art.get(appid, {}))
 
         directory_items.append((run_url, item, False))
 
     return directory_items
 
 
-def create_arts_dictionary(app_entry):
-    """
-    Creates a dictionary of arts keys and their associated links, for a given app entry.
-    :param app_entry: dictionary of app information, containing at least the keys : appid, img_icon_url, img_logo_url
-    :return: dictionary of arts for the app.
-    """
-
-    appid = str(app_entry['appid'])
-    img_icon_url = app_entry['img_icon_url']
-    art_dictionary = {}
-
-    # Multiple fanart https://kodi.wiki/view/Artwork_types#fanart.23
-    SUPPORTED_ART_TYPES = ['poster', 'landscape', 'banner', 'clearlogo', 'thumb', 'fanart', 'fanart1', 'fanart2', 'icon']
-
-    for art_type in SUPPORTED_ART_TYPES:
-        art_dictionary[art_type] = arts.resolve_art_url(art_type, appid, img_icon_url)
-    return art_dictionary
-
-
 def main():
     log('steam-id = ' + ('*' * 17 if __addon__.getSetting('steam-id') else '(not set)'))
     log('steam-key = ' + ('*' * 32 if __addon__.getSetting('steam-key') else '(not set)'))
