synchronization of developers in entries with developers list

code/maintenance.py

@@ -119,7 +119,7 @@ def create_toc(title, file, entries):
     # assemble rows
     rows = []
     for entry in entries:
-        rows.append('- **[{}]({})** ({})'.format(entry['name'], '../' + entry['file'], ', '.join(
+        rows.append('- **[{}]({})** ({})'.format(entry['Name'], '../' + entry['file'], ', '.join(
             entry['code language'] + entry['code license'] + entry['state'])))
 
     # sort rows (by title)
@@ -407,7 +407,7 @@ def update_statistics(infos):
         rel(number_inactive))
 
     if number_inactive > 0:
-        entries_inactive = [(x['name'], x['inactive']) for x in infos if 'inactive' in x]
+        entries_inactive = [(x['Name'], x['inactive']) for x in infos if 'inactive' in x]
         entries_inactive.sort(key=lambda x: str.casefold(x[0]))  # first sort by name
         entries_inactive.sort(key=lambda x: x[1], reverse=True)  # then sort by inactive year (more recently first)
         entries_inactive = ['{} ({})'.format(*x) for x in entries_inactive]
@@ -422,7 +422,7 @@ def update_statistics(infos):
     # number_no_language = sum(1 for x in infois if field not in x)
     # if number_no_language > 0:
     #     statistics += 'Without language tag: {} ({:.1f}%)\n\n'.format(number_no_language, rel(number_no_language))
-    #     entries_no_language = [x['name'] for x in infois if field not in x]
+    #     entries_no_language = [x['Name'] for x in infois if field not in x]
     #     entries_no_language.sort()
     #     statistics += ', '.join(entries_no_language) + '\n\n'
 
@@ -447,7 +447,7 @@ def update_statistics(infos):
     number_no_license = sum(1 for x in infos if field not in x)
     if number_no_license > 0:
         statistics += 'Without license tag: {} ({:.1f}%)\n\n'.format(number_no_license, rel(number_no_license))
-        entries_no_license = [x['name'] for x in infos if field not in x]
+        entries_no_license = [x['Name'] for x in infos if field not in x]
         entries_no_license.sort()
         statistics += ', '.join(entries_no_license) + '\n\n'
 
@@ -491,7 +491,7 @@ def update_statistics(infos):
     entries = []
     for info in infos:
         if 'download' not in info and 'play' not in info:
-            entries.append(info['name'])
+            entries.append(info['Name'])
     entries.sort(key=str.casefold)
     statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
 
@@ -511,7 +511,7 @@ def update_statistics(infos):
                         break
             # if there were repositories, but none popular, add them to the list
             if not popular:
-                entries.append(info['name'])
+                entries.append(info['Name'])
                 # print(info[field])
     entries.sort(key=str.casefold)
     statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
@@ -562,7 +562,7 @@ def update_statistics(infos):
     c_cpp_project_without_build_system = []
     for info in infos:
         if field not in info and ('C' in info['code language'] or 'C++' in info['code language']):
-            c_cpp_project_without_build_system.append(info['name'])
+            c_cpp_project_without_build_system.append(info['Name'])
     c_cpp_project_without_build_system.sort(key=str.casefold)
     statistics += '##### C and C++ projects without build system information ({})\n\n'.format(
         len(c_cpp_project_without_build_system)) + ', '.join(c_cpp_project_without_build_system) + '\n\n'
@@ -572,7 +572,7 @@ def update_statistics(infos):
     for info in infos:
         if field in info and 'CMake' in info[field] and (
                 'C' in info['code language'] or 'C++' in info['code language']):
-            c_cpp_project_not_cmake.append(info['name'])
+            c_cpp_project_not_cmake.append(info['Name'])
     c_cpp_project_not_cmake.sort(key=str.casefold)
     statistics += '##### C and C++ projects with a build system different from CMake ({})\n\n'.format(
         len(c_cpp_project_not_cmake)) + ', '.join(c_cpp_project_not_cmake) + '\n\n'
@@ -615,7 +615,7 @@ def export_json(infos):
     for info in infos:
 
         # game & description
-        entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['name'], info['home'][0],
+        entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['Name'], info['home'][0],
                                                                              r'https://github.com/Trilarion/opensourcegames/blob/master/entries/' +
                                                                              info['file']),
                  textwrap.shorten(info['description'], width=60, placeholder='..')]
@@ -773,10 +773,10 @@ def export_primary_code_repositories_json(infos):
                         continue
 
             if not consumed:
-                unconsumed_entries.append([info['name'], info[field]])
+                unconsumed_entries.append([info['Name'], info[field]])
                 # print output
                 if 'code repository' in info:
-                    print('Entry "{}" unconsumed repo: {}'.format(info['name'], info[field]))
+                    print('Entry "{}" unconsumed repo: {}'.format(info['Name'], info[field]))
 
     # sort them alphabetically (and remove duplicates)
     for k, v in primary_repos.items():
@@ -889,7 +889,7 @@ def update_inspirations(infos):
     # collect information
     originals = {}
     for info in infos:
-        name = info['name']
+        name = info['Name']
         keywords = info['keywords']
         ins = [x[12:] for x in keywords if x.startswith('inspired by ')]
         if ins:
@@ -924,7 +924,7 @@ def update_developer(infos):
     developer = {}
     for info in infos:
         if 'developer' in info:
-            name = info['name']
+            name = info['Name']
             devs = info['developer']
             for dev in devs:
                 if dev in developer:
@@ -954,7 +954,7 @@ def check_code_dependencies(infos):
     valid_dependencies = list(utils.constants.general_code_dependencies_without_entry.keys())
     for info in infos:
         if any((x in ('framework', 'library', 'game engine') for x in info['keywords'])):
-            name = info['name']
+            name = info['Name']
             if name in utils.constants.code_dependencies_aliases:
                 valid_dependencies.extend(utils.constants.code_dependencies_aliases[name])
             else:

code/maintenance_developers.py

@@ -6,13 +6,14 @@ stored Git repositories.
 import os
 import sys
 import requests
+from utils import osg, osg_ui
 from bs4 import BeautifulSoup
 from utils import constants as c, utils, osg, osg_github
 
 
 def developer_info_lookup(name):
     for dev in developer_info:
-        if name == dev['name']:
+        if name == dev['Name']:
             return dev
     return None
 
@@ -21,22 +22,7 @@ def developer_info_lookup(name):
 SF_alias_list = {'Erik Johansson (aka feneur)': 'Erik Johansson', 'Itms': 'Nicolas Auvray',
                  'Wraitii': 'Lancelot de Ferrière', 'Simzer': 'Simon Laszlo', 'armin bajramovic': 'Armin Bajramovic'}
 
-if __name__ == "__main__":
+def test():
-
-    # read developer info
-    developer_info = osg.read_developer_info()
-    osg.write_developer_info(developer_info)  # write again just to make it nice and as sanity check
-
-    sys.exit(0)
-
-
-
-    # assemble info
-    entries = osg.assemble_infos()
-
-    # cross-check
-    osg.compare_entries_developers(entries, developer_info)
-
     # loop over infos
     developers = ''
     try:
@@ -44,7 +30,7 @@ if __name__ == "__main__":
         # active = False
         for entry in entries:
 
-            # if entry['name'] == 'Aleph One':
+            # if entry['Name'] == 'Aleph One':
             #    active = True
             # if not active:
             #    continue
@@ -55,7 +41,7 @@ if __name__ == "__main__":
                 break
 
             # print
-            entry_name = '{} - {}'.format(entry['file'], entry['name'])
+            entry_name = '{} - {}'.format(entry['file'], entry['Name'])
             print(entry_name)
             content = ''
 
@@ -136,3 +122,122 @@ if __name__ == "__main__":
     finally:
         # store developer info
         utils.write_text(os.path.join(c.root_path, 'collected_developer_info.txt'), developers)
+
+def compare_entries_developers(entries, developers):
+    """
+    Cross checks the game entries lists and the developers lists.
+    :param entries: List of game entries
+    :param developers: List of developers
+    """
+
+    # from the entries create a dictionary with developer names
+    devs1 = {}
+    for entry in entries:
+        name = entry['Name']
+        for dev in entry.get('developer', []):
+            if dev in devs1:
+                devs1[dev].append(name)
+            else:
+                devs1[dev] = [name]
+    devs1_names = set(devs1.keys())
+
+    # from the developers create a dictionary with developer names
+    devs2 = dict(zip((dev['Name'] for dev in developers), (dev['Games'] for dev in developers)))
+    devs2_names = set(devs2.keys())
+
+    # devs only in entries
+    for dev in devs1_names - devs2_names:
+        print('Warning: dev "{}" only in entries ({}), not in developers'.format(dev, ','.join(devs1[dev])))
+    # devs only in developers
+    for dev in devs2_names - devs1_names:
+        print('Warning: dev "{}" only in developers ({}), not in entries'.format(dev, ','.join(devs2[dev])))
+    # for those in both, check that the games lists are equal
+    for dev in devs1_names.intersection(devs2_names):
+        games1 = set(devs1[dev])
+        games2 = set(devs2[dev])
+        delta = games1 - games2
+        if delta:
+            print('Warning: dev "{}" has games in entries ({}) that are not present in developers'.format(dev,
+                                                                                                          ', '.join(
+                                                                                                              delta)))
+        delta = games2 - games1
+        if delta:
+            print('Warning: dev "{}" has games in developers ({}) that are not present in entries'.format(dev, delta))
+
+
+class DevelopersMaintainer:
+
+    def __init__(self):
+        self.developers = None
+        self.entries = None
+
+    def read_developer(self):
+        self.developers = osg.read_developers()
+        print('{} developers read'.format(len(self.developers)))
+
+    def write_developer(self):
+        if not self.developers:
+            print('developers not yet loaded')
+            return
+        osg.write_developers(self.developers)
+        print('developers written')
+
+    def check_for_duplicates(self):
+        if not self.developers:
+            print('developers not yet loaded')
+            return
+        developer_names = [x['Name'] for x in self.developers]
+        for index, name in enumerate(developer_names):
+            for other_name in developer_names[index + 1:]:
+                if osg.name_similarity(name, other_name) > 0.8:
+                    print(' {} - {} is similar'.format(name, other_name))
+        print('duplicates checked')
+
+    def check_for_orphans(self):
+        if not self.developers:
+            print('developers not yet loaded')
+            return
+        for dev in self.developers:
+            if not dev['Games']:
+                print(' {} has no "Games" field'.format(dev['Name']))
+        print('orphanes checked')
+        
+    def check_for_missing_developers_in_entries(self):
+        if not self.developers:
+            print('developer not yet loaded')
+            return
+        if not self.entries:
+            print('entries not yet loaded')
+            return
+        for dev in self.developers:
+            dev_name = dev['Name']
+            for entry_name in dev['Games']:
+                x = [x for x in self.entries if x['Title'] == entry_name]
+                assert len(x) <= 1
+                if not x:
+                    print('Entry "{}" listed as game of developer "{}" but this entry does not exist'.format(entry_name, dev_name))
+                else:
+                    entry = x[0]
+                    if 'Developer' not in entry or dev_name not in entry['Developer']:
+                        print('Entry "{}" listed in developer "{}" but not listed in that entry'.format(entry_name, dev_name))
+        print('missed developer checked')
+
+    def read_entries(self):
+        self.entries = osg.read_entries()
+        print('{} entries read'.format(len(self.entries)))
+
+
+if __name__ == "__main__":
+
+    m = DevelopersMaintainer()
+
+    actions = {
+        'Read developers': m.read_developer,
+        'Write developers': m.write_developer,
+        'Check for duplicates': m.check_for_duplicates,
+        'Check for orphans': m.check_for_orphans,
+        'Check for games in developers not listed': m.check_for_missing_developers_in_entries,
+        'Read entries': m.read_entries
+    }
+
+    osg_ui.run_simple_button_app('Maintenance developer', actions)

code/maintenance_inspirations.py

@@ -43,8 +43,8 @@ class InspirationMaintainer:
             print('inspirations not yet loaded')
             return
         for inspiration in self.inspirations.values():
-            if not inspiration['inspired entries']:
+            if not inspiration['Inspired entries']:
-                print(' {} has no inspired entries'.format(inspiration['name']))
+                print(' {} has no inspired entries'.format(inspiration['Name']))
         print('orphanes checked')
 
     def check_for_missing_inspirations_in_entries(self):
@@ -55,18 +55,27 @@ class InspirationMaintainer:
             print('entries not yet loaded')
             return
         for inspiration in self.inspirations.values():
-            inspiration_name = inspiration['name']
+            inspiration_name = inspiration['Name']
-            for entry_name in inspiration['inspired entries']:
+            for entry_name in inspiration['Inspired entries']:
-                x = [x for x in self.entries if x['title'] == entry_name]
+                x = [x for x in self.entries if x['Title'] == entry_name]
                 assert len(x) <= 1
                 if not x:
                     print('Entry "{}" listed in inspiration "{}" but this entry does not exist'.format(entry_name, inspiration_name))
                 else:
                     entry = x[0]
-                    if 'inspirations' not in entry or inspiration_name not in entry['inspirations']:
+                    if 'Inspirations' not in entry or inspiration_name not in entry['Inspirations']:
                         print('Entry "{}" listed in inspiration "{}" but not listed in this entry'.format(entry_name, inspiration_name))
         print('missed inspirations checked')
 
+    def check_for_wikipedia_links(self):
+        if not self.inspirations:
+            print('inspirations not yet loaded')
+            return
+        for inspiration in self.inspirations.values():
+            if 'Media' in inspiration and any(('https://en.wikipedia.org/wiki/' in x for x in inspiration['Media'])):
+                continue
+            # search in wikipedia
+
     def update_inspired_entries(self):
         if not self.inspirations:
             print('inspirations not yet loaded')
@@ -76,15 +85,15 @@ class InspirationMaintainer:
             return
         # loop over all inspirations and delete inspired entries
         for inspiration in self.inspirations.values():
-            inspiration['inspired entries'] = []
+            inspiration['Inspired entries'] = []
         # loop over all entries and add to inspirations of entry
         for entry in self.entries:
-            entry_name = entry['title']
+            entry_name = entry['Title']
-            for inspiration in entry.get('inspirations', []):
+            for inspiration in entry.get('Inspirations', []):
                 if inspiration in self.inspirations:
-                    self.inspirations[inspiration]['inspired entries'].append(entry_name)
+                    self.inspirations[inspiration]['Inspired entries'].append(entry_name)
                 else:
-                    self.inspirations[inspiration] = {'name': inspiration, 'inspired entries': [entry_name]}
+                    self.inspirations[inspiration] = {'Name': inspiration, 'Inspired entries': [entry_name]}
         print('inspired entries updated')
 
     def read_entries(self):

code/utils/constants.py

@@ -12,7 +12,7 @@ tocs_path = os.path.join(entries_path, 'tocs')
 code_path = os.path.join(root_path, 'code')
 
 inspirations_file = os.path.join(root_path, 'inspirations.md')
-developer_file = os.path.join(root_path, 'developer.md')
+developer_file = os.path.join(root_path, 'developers.md')
 
 # local config
 local_config_file = os.path.join(root_path, 'local-config.ini')
@@ -33,12 +33,12 @@ def get_config(key):
 generic_comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
 
 # these fields have to be present in each entry (in this order)
-essential_fields = ('Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
+essential_fields = ('Title', 'Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
 
 # only these fields can be used currently (in this order)
 valid_fields = (
-    'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
+    'Title', 'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
-    'Code license', 'Code dependencies', 'Assets license', 'Developer')
+    'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Note', 'Building')
 
 valid_building_fields = ('Build system', 'Build instructions')
 
@@ -91,7 +91,9 @@ general_code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
                                    'tkinter': 'https://docs.python.org/3/library/tk.html'}
 
 # developer information (in the file all fields will be capitalized)
-valid_developer_fields = ('name', 'games', 'contact', 'organization', 'home')
+essential_developer_fields = ('Name', 'Games')
+valid_developer_fields = essential_developer_fields + ('Contact', 'Home', 'Organization')
 
 # inspiration/original game information (in the file all fields will be capitalized)
-valid_inspiration_fields = ('name', 'inspired entries', 'media')
+essential_inspiration_fields = ('Name', 'Inspired entries')
+valid_inspiration_fields = essential_inspiration_fields + ('Media',)

code/utils/osg.py

@@ -4,104 +4,9 @@ Specific functions working on the games.
 
 import re
 from difflib import SequenceMatcher
-from utils import utils
+from utils import utils, osg_parse
-import lark
-
 from utils.constants import *
 
-
-class ListingTransformer(lark.Transformer):
-    """
-    Transforms content parsed by grammar_listing.lark further.
-    Used for the developer and inspirations list.
-    """
-
-    def unquoted_value(self, x):
-        return x[0].value
-
-    def quoted_value(self, x):
-        return x[0].value[1:-1]  # remove quotation marks
-
-    def property(self, x):
-        """
-        The key of a property will be converted to lower case and the value part is the second part
-        :param x:
-        :return:
-        """
-        return x[0].lower(), x[1:]
-
-    def name(self, x):
-        """
-        The name part is treated as a property with key "name"
-        :param x:
-        :return:
-        """
-        return 'name', x[0].value
-
-    def entry(self, x):
-        """
-        All (key, value) tuples are inserted into a dictionary.
-        :param x:
-        :return:
-        """
-        d = {}
-        for key, value in x:
-            if key in d:
-                raise RuntimeError('Key in entry appears twice')
-            d[key] = value
-        return d
-
-    def start(self, x):
-        return x
-
-
-# transformer
-class EntryTransformer(lark.Transformer):
-
-    def unquoted_value(self, x):
-        return x[0].value
-
-    def quoted_value(self, x):
-        return x[0].value[1:-1]  # remove quotation marks
-
-    def property(self, x):
-        """
-        The key of a property will be converted to lower case and the value part is the second part
-        :param x:
-        :return:
-        """
-        return x[0].lower(), x[1:]
-
-    def title(self, x):
-        return 'title', x[0].value
-
-    def note(self, x):
-        """
-        Optional
-        :param x:
-        :return:
-        """
-        if not x:
-            raise lark.Discard
-        return 'note', ''.join((x.value for x in x))
-
-    def building(self, x):
-        d = {}
-        for key, value in x:
-            if key in d:
-                raise RuntimeError('Key in entry appears twice')
-            d[key] = value
-        return 'building', d
-
-    def start(self, x):
-        d = {}
-        for key, value in x:
-            if key in d:
-                raise RuntimeError('Key in entry appears twice')
-            d[key] = value
-        return d
-
-
 regex_sanitize_name = re.compile(r"[^A-Za-z 0-9-+]+")
 regex_sanitize_name_space_eater = re.compile(r" +")
 
@@ -387,48 +292,32 @@ def extract_links():
     return urls
 
 
-def read_and_parse(content_file: str, grammar_file: str, transformer: lark.Transformer):
+def read_developers():
-    """
-    Reads a content file and a grammar file and parses the content with the grammar following by
-    transforming the parsed output and returning the transformed result.
-    :param content_file:
-    :param grammar_file:
-    :param transformer:
-    :return:
-    """
-    content = utils.read_text(content_file)
-    grammar = utils.read_text(grammar_file)
-    parser = lark.Lark(grammar, debug=False, parser='lalr')
-    tree = parser.parse(content)
-    return transformer.transform(tree)
-
-
-def read_developer_info():
     """
 
     :return:
     """
     grammar_file = os.path.join(code_path, 'grammar_listing.lark')
-    transformer = ListingTransformer()
+    developers = osg_parse.read_and_parse(developer_file, grammar_file, osg_parse.ListingTransformer)
-    developers = read_and_parse(developer_file, grammar_file, transformer)
+
     # now transform a bit more
     for index, dev in enumerate(developers):
         # check for valid keys
         for field in dev.keys():
             if field not in valid_developer_fields:
-                raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['name']))
+                raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['Name']))
         # strip from name or organization (just in case)
-        for field in ('name', ):
+        for field in ('Name', ):
             if field in dev:
                 dev[field] = dev[field].strip()
         # split games, contact (are lists)
-        for field in ('games', 'contact'):
+        for field in ('Games', 'Contact'):
             if field in dev:
                 content = dev[field]
                 content = [x.strip() for x in content]
                 dev[field] = content
     # check for duplicate names entries
-    names = [dev['name'] for dev in developers]
+    names = [dev['Name'] for dev in developers]
     duplicate_names = (name for name in names if names.count(name) > 1)
     duplicate_names = set(duplicate_names)  # to avoid duplicates in duplicate_names
     if duplicate_names:
@@ -436,7 +325,7 @@ def read_developer_info():
     return developers
 
 
-def write_developer_info(developers):
+def write_developers(developers):
     """
 
     :return:
@@ -448,19 +337,19 @@ def write_developer_info(developers):
     content += '# Developer [{}]\n\n'.format(len(developers))
 
     # sort by name
-    developers.sort(key=lambda x: str.casefold(x['name']))
+    developers.sort(key=lambda x: str.casefold(x['Name']))
 
     # iterate over them
     for dev in developers:
         keys = list(dev.keys())
         # developer name
-        content += '## {} [{}]\n\n'.format(dev['name'], len(dev['games']))
+        content += '## {} [{}]\n\n'.format(dev['Name'], len(dev['games']))
-        keys.remove('name')
+        keys.remove('Name')
 
         # all the remaining in alphabetical order, but 'games' first
-        keys.remove('games')
+        keys.remove('Games')
         keys.sort()
-        keys = ['games'] + keys
+        keys = ['Games'] + keys
         for field in keys:
             value = dev[field]
             field = field.capitalize()
@@ -484,34 +373,34 @@ def read_inspirations():
     """
     # read inspirations
 
+    # read and parse inspirations
     grammar_file = os.path.join(code_path, 'grammar_listing.lark')
-    transformer = ListingTransformer()
+    inspirations = osg_parse.read_and_parse(inspirations_file, grammar_file, osg_parse.ListingTransformer)
-    inspirations = read_and_parse(inspirations_file, grammar_file, transformer)
 
     # now inspirations is a list of dictionaries for every entry with keys (valid_developers_fields)
 
     # now transform a bit more
-    for index, inspiration in enumerate(inspirations):
+    for inspiration in inspirations:
         # check that keys are valid keys
         for field in inspiration.keys():
             if field not in valid_inspiration_fields:
-                raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['name']))
+                raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['Name']))
         # split lists
-        for field in ('inspired entries',):
+        for field in ('Inspired entries',):
             if field in inspiration:
                 content = inspiration[field]
                 content = [x.strip() for x in content]
                 inspiration[field] = content
 
     # check for duplicate names entries
-    names = [inspiration['name'] for inspiration in inspirations]
+    names = [inspiration['Name'] for inspiration in inspirations]
     duplicate_names = (name for name in names if names.count(name) > 1)
     duplicate_names = set(duplicate_names)  # to avoid duplicates in duplicate_names
     if duplicate_names:
         raise RuntimeError('Duplicate inspiration names: {}'.format(', '.join(duplicate_names)))
 
     # convert to dictionary
-    inspirations = {x['name']: x for x in inspirations}
+    inspirations = {x['Name']: x for x in inspirations}
 
     return inspirations
 
@@ -532,25 +421,25 @@ def write_inspirations(inspirations):
     content += '# Inspirations [{}]\n\n'.format(len(inspirations))
 
     # sort by name
-    inspirations.sort(key=lambda x: str.casefold(x['name']))
+    inspirations.sort(key=lambda x: str.casefold(x['Name']))
 
     # iterate over them
     for inspiration in inspirations:
         keys = list(inspiration.keys())
         # inspiration name
-        content += '## {} [{}]\n\n'.format(inspiration['name'], len(inspiration['inspired entries']))
+        content += '## {} [{}]\n\n'.format(inspiration['Name'], len(inspiration['Inspired entries']))
-        keys.remove('name')
+        keys.remove('Name')
 
         # all the remaining in alphabetical order, but "inspired entries" first
-        keys.remove('inspired entries')
+        keys.remove('Inspired entries')
         keys.sort()
-        keys = ['inspired entries'] + keys
+        keys = ['Inspired entries'] + keys
         for field in keys:
             value = inspiration[field]
             field = field.capitalize()
             # lists get special treatment
             if isinstance(value, list):
-                value.sort(key=str.casefold) # sorted alphabetically
+                value.sort(key=str.casefold)  # sorted alphabetically
                 value = [x if not ',' in x else '"{}"'.format(x) for x in value]  # surround those with a comma with quotation marks
                 value = ', '.join(value)
             content += '- {}: {}\n'.format(field, value)
@@ -565,13 +454,10 @@ def read_entries():
     Parses all entries and assembles interesting infos about them.
     """
 
-    # setup parser
+    # setup parser and transformer
     grammar_file = os.path.join(code_path, 'grammar_entries.lark')
     grammar = utils.read_text(grammar_file)
-    parser = lark.Lark(grammar, debug=False, parser='lalr')
+    parse = osg_parse.create(grammar, osg_parse.EntryTransformer)
-
-    # setup transformer
-    transformer = EntryTransformer()
 
     # a database of all important infos about the entries
     entries = []
@@ -585,13 +471,13 @@ def read_entries():
 
         # parse and transform entry content
         try:
-            tree = parser.parse(content)
+            entry = parse(content)
-
-            entry = transformer.transform(tree)
             # add file information
-            entry['file'] = file
+            entry['File'] = file
 
             check_entry(entry)
+
+            post_process(entry)
         except Exception as e:
             print('{} - {}'.format(file, e))
             exception_happened = True
@@ -604,6 +490,22 @@ def read_entries():
 
     return entries
 
+def post_process(entry):
+    """
+
+    :param entry:
+    :return:
+    """
+
+    # remove all parentheses from developers
+    if 'Developer' in entry:
+        devs = entry['Developer']
+        devs = [re.sub(r'\([^)]*\)', '', x).strip() for x in devs]
+        if any(not x for x in devs):
+            raise RuntimeError('Empty developer')
+        entry['Developer'] = devs
+
+
 
 def check_entry(entry):
     """
@@ -613,14 +515,19 @@ def check_entry(entry):
     """
     message = ''
 
-    file = entry['file']
+    file = entry['File']
 
     # check canonical file name
-    canonical_file_name = canonical_entry_name(entry['title']) + '.md'
+    canonical_file_name = canonical_entry_name(entry['Title']) + '.md'
     # we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
     if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
         message += 'file name should be {}\n'.format(canonical_file_name)
 
+    # check for essential fields
+    for field in essential_fields:
+        if field not in entry:
+            message += 'essential property "{}" missing\n'.format(field)
+
     if message:
         raise RuntimeError(message)
 
@@ -661,20 +568,19 @@ def create_entry_content(entry):
     """
 
     # title
-    content = '# {}\n\n'.format(entry['title'])
+    content = '# {}\n\n'.format(entry['Title'])
 
     # now properties in the recommended order
     for field in valid_fields:
-        field_name = field.lower()
+        if field in entry:
-        if field_name in entry:
+            c = entry[field]
-            c = entry[field_name]
             c = ['"{}"'.format(x) if ',' in x else x for x in c]
             content += '- {}: {}\n'.format(field, ', '.join(c))
     content += '\n'
 
     # if there is a note, insert it
-    if 'note' in entry:
+    if 'Note' in entry:
-        content += entry['note']
+        content += entry['Note']
 
     # building header
     content += '## Building\n'
@@ -682,59 +588,15 @@ def create_entry_content(entry):
     # building properties if present
     has_properties = False
     for field in valid_building_fields:
-        field_name = field.lower()
+        if field in entry['Building']:
-        if field_name in entry['building']:
             if not has_properties:
                 has_properties = True
                 content += '\n'
-            content += '- {}: {}\n'.format(field, ', '.join(entry['building'][field_name]))
+            content += '- {}: {}\n'.format(field, ', '.join(entry['Building'][field]))
 
     # if there is a note, insert it
-    if 'note' in entry['building']:
+    if 'Note' in entry['Building']:
         content += '\n'
-        content += entry['building']['note']
+        content += entry['Building']['Note']
 
-    return content
+    return content
-
-def compare_entries_developers(entries, developers):
-    """
-    Cross checks the game entries lists and the developers lists.
-    :param entries: List of game entries
-    :param developers: List of developers
-    """
-
-    # from the entries create a dictionary with developer names
-    devs1 = {}
-    for entry in entries:
-        name = entry['name']
-        for dev in entry.get('developer', []):
-            if dev in devs1:
-                devs1[dev].append(name)
-            else:
-                devs1[dev] = [name]
-    devs1_names = set(devs1.keys())
-
-    # from the developers create a dictionary with developer names
-    devs2 = dict(zip((dev['name'] for dev in developers), (dev['games'] for dev in developers)))
-    devs2_names = set(devs2.keys())
-
-    # devs only in entries
-    for dev in devs1_names - devs2_names:
-        print('Warning: dev "{}" only in entries ({}), not in developers'.format(dev, ','.join(devs1[dev])))
-    # devs only in developers
-    for dev in devs2_names - devs1_names:
-        print('Warning: dev "{}" only in developers ({}), not in entries'.format(dev, ','.join(devs2[dev])))
-    # for those in both, check that the games lists are equal
-    for dev in devs1_names.intersection(devs2_names):
-        games1 = set(devs1[dev])
-        games2 = set(devs2[dev])
-        delta = games1 - games2
-        if delta:
-            print('Warning: dev "{}" has games in entries ({}) that are not present in developers'.format(dev,
-                                                                                                          ', '.join(
-                                                                                                              delta)))
-        delta = games2 - games1
-        if delta:
-            print('Warning: dev "{}" has games in developers ({}) that are not present in entries'.format(dev,
-                                                                                                          ', '.join(
-                                                                                                              delta)))

code/utils/osg_parse.py

@@ -0,0 +1,140 @@
+"""
+
+"""
+
+from functools import partial
+import lark
+from utils import utils, constants as c
+
+
+class ListingTransformer(lark.Transformer):
+    """
+    Transforms content parsed by grammar_listing.lark further.
+    Used for the developer and inspirations list.
+    """
+
+    def unquoted_value(self, x):
+        return x[0].value
+
+    def quoted_value(self, x):
+        return x[0].value[1:-1]  # remove quotation marks
+
+    def property(self, x):
+        """
+        The key of a property will be converted to lower case and the value part is the second part
+        :param x:
+        :return:
+        """
+        return x[0], x[1:]
+
+    def name(self, x):
+        """
+        The name part is treated as a property with key "Name"
+        :param x:
+        :return:
+        """
+        return 'Name', x[0].value
+
+    def entry(self, x):
+        """
+        All (key, value) tuples are inserted into a dictionary.
+        :param x:
+        :return:
+        """
+        d = {}
+        for key, value in x:
+            if key in d:
+                raise RuntimeError('Key in entry appears twice')
+            d[key] = value
+        return d
+
+    def start(self, x):
+        return x
+
+
+# transformer
+class EntryTransformer(lark.Transformer):
+
+    def unquoted_value(self, x):
+        return x[0].value
+
+    def quoted_value(self, x):
+        return x[0].value[1:-1]  # remove quotation marks
+
+    def property(self, x):
+        """
+        The key of a property will be converted to lower case and the value part is the second part
+        :param x:
+        :return:
+        """
+        return x[0], x[1:]
+
+    def title(self, x):
+        return 'Title', x[0].value
+
+    def note(self, x):
+        """
+        Optional
+        :param x:
+        :return:
+        """
+        if not x:
+            raise lark.Discard
+        return 'Note', ''.join((x.value for x in x))
+
+    def building(self, x):
+        d = {}
+        for key, value in x:
+            if key in d:
+                raise RuntimeError('Key in entry appears twice')
+            d[key] = value
+        return 'Building', d
+
+    def start(self, x):
+        # we do the essential fields and valid fields checks right here
+        fields = [x[0] for x in x]
+        # check for essential fields
+        for field in c.essential_fields:
+            if field not in fields:
+                raise RuntimeError('Essential field "{}" is missing'.format(field))
+        # check for valid fields (in that order)
+        index = 0
+        for field in fields:
+            while index < len(c.valid_fields) and field != c.valid_fields[index]:
+                index += 1
+            if index == len(c.valid_fields):
+                raise RuntimeError('Field "{}" either not valid or in wrong order'.format(field))
+        d = {}
+        for key, value in x:
+            if key in d:
+                raise RuntimeError('Key in entry appears twice')
+            d[key] = value
+        return d
+
+
+def parse(parser, transformer, content):
+    tree = parser.parse(content)
+    value = transformer.transform(tree)
+    return value
+
+
+def create(grammar, Transformer):
+    parser = lark.Lark(grammar, debug=False, parser='lalr')
+    transformer = Transformer()
+    return partial(parse, parser, transformer)
+
+
+def read_and_parse(content_file: str, grammar_file: str, Transformer: lark.Transformer):
+    """
+    Reads a content file and a grammar file and parses the content with the grammar following by
+    transforming the parsed output and returning the transformed result.
+    :param content_file:
+    :param grammar_file:
+    :param transformer:
+    :return:
+    """
+    grammar = utils.read_text(grammar_file)
+    parse = create(grammar, Transformer)
+
+    content = utils.read_text(content_file)
+    return parse(content)

code/utils/osg_wikipedia.py

@@ -0,0 +1,16 @@
+"""
+Everything specific to search in Wikipedia.
+Using https://github.com/goldsmith/Wikipedia
+"""
+
+import wikipedia
+
+
+def search(search_term, results=3):
+    """
+
+    :param search_term:
+    :param max_results:
+    :return:
+    """
+    return wikipedia.search(search_term, results=results)

developers.md

@@ -32,7 +32,7 @@
 
 ## Alex Margarit [1]
 
-- Games: a2x
+- Games: faur
 
 ## Alexander Lang [1]
 

entries/shotgun_debugger.md

@@ -10,7 +10,7 @@
 - Code dependencies: SDL
 - Developer: Matthew Sarnoff (Game Creation Society), Chris DeLeon (Game Creation Society), John Nesky (Game Creation Society), Gregory Peng (Game Creation Society), Jeff Thoene (Game Creation Society), Tuscan Knox (music, Game Creation Society), Michael Weber (Game Creation Society)
 
-Shooting game which uses a 3d engine but allows 2d gameplay.
+Shooting game which uses a 3D engine but allows 2D gameplay.
 
 ## Building