Package horizons :: Package util :: Module yamlcache
[hide private]
[frames] | no frames]

Source Code for Module horizons.util.yamlcache

  1  # ################################################### 
  2  # Copyright (C) 2008-2017 The Unknown Horizons Team 
  3  # team@unknown-horizons.org 
  4  # This file is part of Unknown Horizons. 
  5  # 
  6  # Unknown Horizons is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or 
  9  # (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 14  # GNU General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the 
 18  # Free Software Foundation, Inc., 
 19  # 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA 
 20  # ################################################### 
 21   
 22  import logging 
 23  import os 
 24  import threading 
 25  from typing import Optional 
 26   
 27  import yaml 
 28   
 29  from horizons.constants import BUILDINGS, PATHS, RES, TIER, UNITS 
 30  from horizons.util.yamlcachestorage import YamlCacheStorage 
 31   
 32  try: 
 33          from yaml import CSafeLoader as SafeLoader # type: ignore 
 34  except ImportError: 
 35          from yaml import SafeLoader # type: ignore 
36 37 38 # make SafeLoader allow unicode 39 -def construct_yaml_str(self, node):
40 return self.construct_scalar(node)
41 42 43 SafeLoader.add_constructor('tag:yaml.org,2002:python/unicode', construct_yaml_str) 44 SafeLoader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str)
45 46 47 -def parse_token(token, token_klass):
48 """Helper function that tries to parse a constant name. 49 Does not do error detection, but passes unparseable stuff through. 50 Allowed values: integer or token_klass.LIKE_IN_CONSTANTS 51 @param token_klass: "TIER", "RES", "UNITS" or "BUILDINGS" 52 """ 53 classes = {'TIER': TIER, 'RES': RES, 'UNITS': UNITS, 'BUILDINGS': BUILDINGS} 54 55 if not isinstance(token, str): 56 # Probably numeric already 57 return token 58 if not token.startswith(token_klass): 59 # No need to parse anything 60 return token 61 try: 62 return getattr(classes[token_klass], token.split(".", 2)[1]) 63 except AttributeError as e: # token not defined here 64 err = ("This means that you either have to add an entry in horizons/constants.py " 65 "in the class {0!s} for {1!s},\nor {2!s} is actually a typo.". 66 format(token_klass, token, token)) 67 raise Exception(str(e) + "\n\n" + err + "\n")
68
69 70 -def convert_game_data(data):
71 """Translates convenience symbols into actual game data usable by machines""" 72 if isinstance(data, dict): 73 return dict([convert_game_data(i) for i in data.items()]) 74 elif isinstance(data, (tuple, list)): 75 return type(data)((convert_game_data(i) for i in data)) 76 else: # leaf 77 data = parse_token(data, "TIER") 78 data = parse_token(data, "RES") 79 data = parse_token(data, "UNITS") 80 data = parse_token(data, "BUILDINGS") 81 return data
82
83 84 -class YamlCache:
85 """Loads and caches YAML files in a persistent cache. 86 Threadsafe. 87 88 Use get_file for files to cache (default case) or load_yaml_data for special use cases (behaves like yaml.load). 89 """ 90 91 cache = None # type: Optional[YamlCacheStorage] 92 cache_filename = os.path.join(PATHS.CACHE_DIR, 'yamldata.cache') 93 94 sync_scheduled = False 95 96 lock = threading.Lock() 97 98 log = logging.getLogger("yamlcache") 99 100 @classmethod
101 - def load_yaml_data(cls, string_or_stream):
102 """Use this instead of yaml.load everywhere in uh in case get_file isn't useable""" 103 return yaml.load(string_or_stream, Loader=SafeLoader)
104 105 @classmethod
106 - def get_file(cls, filename, game_data=False):
107 """Get contents of a yaml file 108 @param filename: path to the file 109 @param game_data: Whether this file contains data like BUILDINGS.LUMBERJACK to resolve 110 """ 111 with open(filename, 'r', encoding="utf-8") as f: 112 filedata = f.read() 113 114 # calc the hash 115 h = hash(filedata) 116 117 # check for updates or new files 118 if cls.cache is None: 119 cls._open_cache() 120 121 yaml_file_in_cache = (filename in cls.cache and cls.cache[filename][0] == h) 122 if not yaml_file_in_cache: 123 data = cls.load_yaml_data(filedata) 124 if game_data: # need to convert some values 125 try: 126 data = convert_game_data(data) 127 except Exception as e: 128 # add info about file 129 to_add = "\nThis error happened in {0!s} .".format(filename) 130 e.args = (e.args[0] + to_add, ) + e.args[1:] 131 e.message = (e.message + to_add) 132 raise 133 134 cls.lock.acquire() 135 cls.cache[filename] = (h, data) 136 if not cls.sync_scheduled: 137 cls.sync_scheduled = True 138 from horizons.extscheduler import ExtScheduler 139 ExtScheduler().add_new_object(cls._do_sync, cls, run_in=1) 140 cls.lock.release() 141 142 return cls.cache[filename][1] # returns an object from the YAML
143 144 @classmethod
145 - def _open_cache(cls):
146 cls.lock.acquire() 147 cls.cache = YamlCacheStorage.open(cls.cache_filename) 148 cls.lock.release()
149 150 @classmethod
151 - def _do_sync(cls):
152 """Only write to disc once in a while, it's too slow when done every time""" 153 cls.lock.acquire() 154 cls.sync_scheduled = False 155 cls.cache.sync() 156 cls.lock.release()
157