change ValueWithComment from composition to Inheritance (simplifies code a lot)

code/archives.json

@@ -702,6 +702,7 @@
   "https://github.com/egordorichev/LastTry.git",
   "https://github.com/eguneys/lose-your-marbles.git",
   "https://github.com/eguneys/supaxl.git",
+  "https://github.com/ekolis/FrEee.git",
   "https://github.com/electronicarts/CnC_Remastered_Collection.git",
   "https://github.com/ellisonleao/clumsy-bird.git",
   "https://github.com/elnormous/ouzel.git",

code/backlog.txt

@@ -44,9 +44,11 @@ https://github.com/gecko0307/dagon
 https://github.com/googleforgames/open-match
 https://github.com/hiitiger/goverlay
 https://github.com/jmorton06/Lumos
+https://github.com/loudsmilestudios/TetraForce
 https://github.com/open-telemetry/opentelemetry-cpp
 https://github.com/QodotPlugin/qodot-plugin
 https://github.com/Razakhel/RaZ
+https://github.com/Relintai/broken_seals
 https://github.com/samdauwe/BabylonCpp
 https://github.com/SasLuca/rayfork
 https://github.com/Zylann/godot_heightmap_plugin
@@ -96,6 +98,4 @@ https://www.seul.org/~grumbel/tmp/clanlib/games.html
 https://www.tapatalk.com/groups/imperilist/
 https://www.wurfelengine.net/
 https://zdoom.org/downloads (gzdoom, lzdoom)
-https://zope.readthedocs.io/en/latest/
+https://zope.readthedocs.io/en/latest/
-https://github.com/loudsmilestudios/TetraForce
-https://github.com/Relintai/broken_seals

code/github_import.py

@@ -82,7 +82,7 @@ def github_import():
             # read entry
             entry = osg.read_entry(file)
             code_repositories = entry['Code repository']
-            repos = [x.value for x in code_repositories if x.startswith(prefix)]
+            repos = [x for x in code_repositories if x.startswith(prefix)]
             repos[0] += ' @add'
             repos = [x for x in repos if '@add' in x]
             repos = [x.split(' ')[0] for x in repos]
@@ -112,7 +112,7 @@ def github_import():
 
                 # update comment
                 for r in code_repositories:
-                    if r.value.startswith(repo):
+                    if r.startswith(repo):
                         break
                 comments = r.comment
                 if comments:
@@ -128,7 +128,7 @@ def github_import():
                 language = info['language']
                 language = language_aliases.get(language, language)
                 if language and language not in entry['Code language'] and language not in ignored_languages:
-                    entry['Code language'].append(osg_parse.ValueWithComment(language))
+                    entry['Code language'].append(language)
                     print('  added to languages: {}'.format(language))
 
                 # contributors
@@ -155,7 +155,7 @@ def github_import():
                     # look up author in entry developers
                     if name not in entry.get('Developer', []):
                         print('   dev "{}" added to entry {}'.format(name, file))
-                        entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(name)]
+                        entry['Developer'] = entry.get('Developer', []) + [name]
 
                     # look up author in developers data base
                     if name in all_developers:
@@ -204,7 +204,7 @@ def github_starring_synchronization():
 
         # get repos
         code_repositories = entry.get('Code repository', [])
-        repos = [x.value for x in code_repositories if x.startswith(prefix)]
+        repos = [x for x in code_repositories if x.startswith(prefix)]
         repos[0] += ' @add'
         repos = [x for x in repos if '@add' in x]
         repos = [x.split(' ')[0] for x in repos]

code/gitlab_import.py

@@ -54,7 +54,7 @@ def gitlab_import():
             # read entry
             entry = osg.read_entry(file)
             code_repositories = entry['Code repository']
-            repos = [x.value for x in code_repositories if x.startswith(prefix)]
+            repos = [x for x in code_repositories if x.startswith(prefix)]
             repos[0] += ' @add'
             repos = [x for x in repos if '@add' in x]
             repos = [x.split(' ')[0] for x in repos]
@@ -77,7 +77,7 @@ def gitlab_import():
 
                 # search for repository
                 for r in code_repositories:
-                    if r.value.startswith(repo):
+                    if r.startswith(repo):
                         break
 
                 # update comment
@@ -94,7 +94,7 @@ def gitlab_import():
                 # language in languages
                 for language, usage in info['languages'].items():
                     if language in c.known_languages and usage > 5 and language not in entry['Code language']:
-                        entry['Code language'].append(osg_parse.ValueWithComment(language))
+                        entry['Code language'].append(language)
                         print('  added to languages: {}'.format(language))
 
             entry['Code repository'] = code_repositories
@@ -125,7 +125,7 @@ def gitlab_starring_synchronization():
 
         # get repos
         code_repositories = entry.get('Code repository', [])
-        repos = [x.value for x in code_repositories if x.startswith(prefix)]
+        repos = [x for x in code_repositories if x.startswith(prefix)]
         repos[0] += ' @add'
         repos = [x for x in repos if '@add' in x]
         repos = [x.split(' ')[0] for x in repos]

code/html/generate_static_website.py

@@ -80,7 +80,7 @@ import math
 import datetime
 import time
 from functools import partial
-from utils import osg, constants as c, utils
+from utils import osg, constants as c, utils, osg_statistics as stat, osg_parse
 from jinja2 import Environment, FileSystemLoader
 import html5lib
 
@@ -97,13 +97,13 @@ alphabet_replacements = {'Á': 'A', 'Å': 'A', 'Ä': 'A', 'É': 'E', 'ⴹ': 'E',
 
 # the subfolder structure
 games_path = ['games']
-frameworks_path = ['frameworks']
+non_games_path = ['frameworks']
 inspirations_path = ['inspirations']
 developers_path = ['developers']
 
 # derived paths
 games_index_path = games_path + ['index.html']
-frameworks_index_path = frameworks_path + ['index.html']
+non_games_index_path = non_games_path + ['index.html']
 inspirations_index_path = inspirations_path + ['index.html']
 developers_index_path = developers_path + ['index.html']
 
@@ -155,11 +155,12 @@ genre_icon_map = {
 # cross-references to the langue pages
 code_language_references = {l: games_by_language_path[:-1] + ['{}#{}'.format(games_by_language_path[-1], osg.canonical_name(l))] for l in c.known_languages}
 
-# map of internal framework names to display names (which are in plural)
+# map of internal non game names to display names (which are in plural)
-framework_names = {
+non_game_category_names = {
     'tool': 'Tools',
     'framework': 'Frameworks',
-    'library': 'Libraries'
+    'library': 'Libraries',
+    'game engine': 'Game Engine'
 }
 
 # we check the output html structure every time
@@ -284,15 +285,18 @@ def url_to(current, target, info=None):
     return url
 
 
-def preprocess(list, key, url):
+def preprocess(items, key, url):
     """
+    Sets a few additional fields in the entries, inspirations, developers in order to generate the right content from
+    them later.
 
-    :param list:
+    :param url:
+    :param items:
     :param key:
     :return:
     """
-    _ = set()
+    _ = set() # this is just to avoid duplicating anchors
-    for item in list:
+    for item in items:
         # add unique anchor ref
         anchor = osg.canonical_name(item[key])
         while anchor in _:
@@ -311,6 +315,11 @@ def preprocess(list, key, url):
 
 
 def game_index(entry):
+    """
+
+    :param entry:
+    :return:
+    """
     e = {
         'url': make_url(entry['href'], entry['Title']),
         'anchor-id': entry['anchor-id']
@@ -326,6 +335,11 @@ def game_index(entry):
 
 
 def inspiration_index(inspiration):
+    """
+
+    :param inspiration:
+    :return:
+    """
     e = {
         'url': make_url(inspiration['href'], inspiration['Name']),
         'anchor-id': inspiration['anchor-id'],
@@ -337,6 +351,11 @@ def inspiration_index(inspiration):
 
 
 def developer_index(developer):
+    """
+
+    :param developer:
+    :return:
+    """
     e = {
         'url': make_url(developer['href'], developer['Name']),
         'anchor-id': developer['anchor-id']
@@ -348,7 +367,12 @@ def developer_index(developer):
 
 
 def shortcut_url(url, name):
+    """
 
+    :param url:
+    :param name:
+    :return:
+    """
     # remove slash at the end
     if url.endswith('/'):
         url = url[:-1]
@@ -388,6 +412,14 @@ def shortcut_url(url, name):
 
 
 def make_url(href, content, title=None, css_class=None):
+    """
+
+    :param href:
+    :param content:
+    :param title:
+    :param css_class:
+    :return:
+    """
     if isinstance(content, str):
         content = make_text(content)
     url = {
@@ -403,9 +435,15 @@ def make_url(href, content, title=None, css_class=None):
 
 
 def make_repo_url(x, name):
+    """
+
+    :param x:
+    :param name:
+    :return:
+    """
     # parse comments
     comments = []
-    if x.has_comment():
+    if isinstance(x, osg_parse.Value):
         for c in x.comment.split(','):
             c = c.strip()
             if not c.startswith('@'):
@@ -427,7 +465,7 @@ def make_repo_url(x, name):
                 else:
                     comments.append(make_icon('star-o', 'low rated'))
     # this is the default element
-    url = make_url(x.value, shortcut_url(x.value, name))
+    url = make_url(x, shortcut_url(x, name))
     if comments:
         return make_enumeration([url, make_enclose(make_enumeration(comments), '(', ')')], '')
     else:
@@ -435,6 +473,12 @@ def make_repo_url(x, name):
 
 
 def make_icon(css_class, title=None):
+    """
+
+    :param css_class:
+    :param title:
+    :return:
+    """
     icon = {
         'type': 'icon',
         'class': css_class,
@@ -445,6 +489,12 @@ def make_icon(css_class, title=None):
 
 
 def make_text(content, css_class=None):
+    """
+
+    :param content:
+    :param css_class:
+    :return:
+    """
     text = {
         'type': 'text',
         'text': content
@@ -454,13 +504,14 @@ def make_text(content, css_class=None):
     return text
 
 
-def make_nothing():
-    return {
-        'type': 'nothing'
-    }
-
-
 def make_enclose(entry, left, right):
+    """
+
+    :param entry:
+    :param left:
+    :param right:
+    :return:
+    """
     enclose = {
         'type': 'enclose',
         'entry': entry,
@@ -471,6 +522,12 @@ def make_enclose(entry, left, right):
 
 
 def make_enumeration(entries, divider=', '):
+    """
+
+    :param entries:
+    :param divider:
+    :return:
+    """
     enumeration = {
         'type': 'enumeration',
         'entries': entries,
@@ -480,6 +537,11 @@ def make_enumeration(entries, divider=', '):
 
 
 def make_tags(entries):
+    """
+
+    :param entries:
+    :return:
+    """
     return {
         'type': 'tags',
         'enumeration': make_enumeration(entries, divider='')
@@ -505,6 +567,12 @@ def developer_profile_link(link):
 
 
 def convert_inspirations(inspirations, entries):
+    """
+
+    :param inspirations:
+    :param entries:
+    :return:
+    """
     entries_references = {entry['Title']: entry['href'] for entry in entries}
     for inspiration in inspirations:
         name = inspiration['Name']
@@ -524,6 +592,12 @@ def convert_inspirations(inspirations, entries):
 
 
 def convert_developers(developers, entries):
+    """
+
+    :param developers:
+    :param entries:
+    :return:
+    """
     entries_references = {entry['Title']:entry['href'] for entry in entries}
     for developer in developers:
         name = developer['Name']
@@ -552,9 +626,14 @@ def convert_developers(developers, entries):
 
 
 def create_keyword_tag(keyword):
+    """
+
+    :param keyword:
+    :return:
+    """
     if keyword in c.recommended_keywords:
         if keyword in c.non_game_keywords:
-            url = frameworks_index_path.copy()
+            url = non_games_index_path.copy()
         else:
             url = games_by_genres_path.copy()
         url[-1] += '#{}'.format(keyword)
@@ -567,6 +646,11 @@ def create_keyword_tag(keyword):
 
 
 def create_state_texts(states):
+    """
+
+    :param states:
+    :return:
+    """
     texts = []
     if 'mature' in states:
         texts.append(make_text('mature', 'has-text-weight-bold'))
@@ -581,6 +665,13 @@ def create_state_texts(states):
 
 
 def convert_entries(entries, inspirations, developers):
+    """
+
+    :param entries:
+    :param inspirations:
+    :param developers:
+    :return:
+    """
     inspirations_references = {inspiration['Name']: inspiration['href'] for inspiration in inspirations}
     developer_references = {developer['Name']: developer['href'] for developer in developers}
     for entry in entries:
@@ -596,7 +687,7 @@ def convert_entries(entries, inspirations, developers):
             entry['note'] = make_text(entry['Note'], 'is-italic')
 
         # keywords as tags
-        e = [create_keyword_tag(x.value) for x in entry['Keyword']]
+        e = [create_keyword_tag(x) for x in entry['Keyword']]
         entry['keyword'] = make_tags(e)
 
         # other normal fields (not technical info)
@@ -604,8 +695,6 @@ def convert_entries(entries, inspirations, developers):
             if field in entry:
                 e = entry[field]
                 divider = ', '
-                if isinstance(e[0], osg.osg_parse.ValueWithComment):
-                    e = [x.value for x in e]
                 if field == 'Inspiration':
                     e = [make_url(inspirations_references[x], make_text(x)) for x in e]
                 elif field == 'Developer':
@@ -627,8 +716,6 @@ def convert_entries(entries, inspirations, developers):
         # platforms
         if 'Platform' in entry:
             e = entry['Platform']
-            if isinstance(e[0], osg.osg_parse.ValueWithComment):
-                e = [x.value for x in e]
         else:
             e = ['Unspecified']
         e = [make_url(games_by_platform_path[:-1] + ['{}#{}'.format(games_by_platform_path[-1], x.lower())], make_icon(platform_icon_map[x]), x) if x in platform_icon_map else make_text(x) for x in e]
@@ -643,8 +730,6 @@ def convert_entries(entries, inspirations, developers):
                 divider = ', '
                 if not e:
                     continue
-                if isinstance(e[0], osg.osg_parse.ValueWithComment) and field != 'Code repository':
-                    e = [x.value for x in e]
                 if field == 'Code repository':
                     e = [make_repo_url(x, name) for x in e]
                 elif field == 'Code language':
@@ -663,8 +748,6 @@ def convert_entries(entries, inspirations, developers):
         if field in entry['Building']:
             e = entry['Building'][field]
             divider = ', '
-            if isinstance(e[0], osg.osg_parse.ValueWithComment):
-                e = [x.value for x in e]
             e = [make_url(c.build_system_urls[x], x, css_class='is-size-7') if x in c.build_system_urls else make_text(x, 'is-size-7') for x in e]
             namex = make_text('{}: '.format(field), 'is-size-7')
             entry[field.lower()] = [namex, make_enumeration(e, divider)]
@@ -673,19 +756,29 @@ def convert_entries(entries, inspirations, developers):
 
 
 def add_license_links_to_entries(entries):
+    """
+
+    :param entries:
+    :return:
+    """
     for entry in entries:
         licenses = entry['Code license']
-        licenses = [(c.license_urls.get(license.value, ''), license.value) for license in licenses]
+        licenses = [(c.license_urls.get(license, ''), license) for license in licenses]
         entry['Code license'] = licenses
 
 
 def get_top50_games(games):
+    """
+
+    :param games:
+    :return:
+    """
     top50_games = []
     for game in games:
         # get stars of repositories
         stars = 0
         for repo in game.get('Code repository', []):
-            if repo.has_comment():
+            if isinstance(repo, osg_parse.Value):
                 for c in repo.comment.split(','):
                     c = c.strip()
                     if not c.startswith('@'):
@@ -707,25 +800,24 @@ def get_top50_games(games):
 
 def generate(entries, inspirations, developers):
     """
-
+    Regenerates the whole static website given an already imported set of entries, inspirations and developers.
-    :param entries:
+    These datasets must be valid for each other, i.e. each inspiration listed in entries must also have an
-    :param inspirations:
+    entry in inspirations and the same holds for developers.
-    :param developers:
     """
 
-    # split entries in games and frameworks
+    # split entries in games and non-games
-    games, frameworks = [], []
+    games, non_games = [], []
     for entry in entries:
-        (games, frameworks)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry)
+        (games, non_games)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry)
 
     # preprocess
     preprocess(games, 'Title', games_path)
-    preprocess(frameworks, 'Title', frameworks_path)
+    preprocess(non_games, 'Title', non_games_path)
     # TODO preprocess doesn't set the urls for frameworks correctly fix here, do better later
-    for framework in frameworks:
+    for non_game in non_games:
-        keyword = [keyword for keyword in c.non_game_keywords if keyword in framework['Keyword']][0]
+        keyword = [keyword for keyword in c.non_game_keywords if keyword in non_game['Keyword']][0]
-        framework['href'] = frameworks_path + ['{}.html#{}'.format(keyword, framework['anchor-id'])]
+        non_game['href'] = non_games_path + ['{}.html#{}'.format(keyword, non_game['anchor-id'])]
-    entries = games + frameworks
+    entries = games + non_games
     preprocess(inspirations, 'Name', inspirations_path)
     preprocess(developers, 'Name', developers_path)
 
@@ -733,7 +825,7 @@ def generate(entries, inspirations, developers):
     convert_inspirations(inspirations, entries)
     convert_developers(developers, entries)
     convert_entries(games, inspirations, developers)
-    convert_entries(frameworks, inspirations, developers)
+    convert_entries(non_games, inspirations, developers)
 
     # set external links up
     add_license_links_to_entries(games)
@@ -749,7 +841,7 @@ def generate(entries, inspirations, developers):
     games_by_genre = sort_into_categories(games, genres, lambda item, category: category.lower() in item['Keyword'])
     games_by_platform = sort_into_categories(entries, c.valid_platforms, lambda item, category: category in item.get('Platform', []), 'Unspecified')
     games_by_language = sort_into_categories(entries, c.known_languages, lambda item, category: category in item['Code language'])
-    frameworks_by_type = sort_into_categories(frameworks, c.non_game_keywords, lambda item, category: category in item['Keyword'])
+    non_games_by_type = sort_into_categories(non_games, c.non_game_keywords, lambda item, category: category in item['Keyword'])
 
     # extract top 50 Github stars games
     top50_games = get_top50_games(games)
@@ -783,19 +875,19 @@ def generate(entries, inspirations, developers):
 
     # index.html
     base['active_nav'] = 'index'
-    index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(frameworks))) }
+    index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(non_games))) }
     template = environment.get_template('index.jinja')
     write(template.render(index=index), ['index.html'])
 
-    # contribute.html
+    # contribute page
     base['active_nav'] = 'contribute'
     template = environment.get_template('contribute.jinja')
     write(template.render(), ['contribute.html'])
 
-    # statistics
+    # statistics page
     base['active_nav'] = 'statistics'
 
-    # preparation
+    # statistics preparation
     template = environment.get_template('statistics.jinja')
     data = {
         'title': 'Statistics',
@@ -803,48 +895,42 @@ def generate(entries, inspirations, developers):
     }
 
     # build-systems
-    build_systems = []
+    build_systems_stat = stat.get_build_systems(entries)
-    field = 'Build system'
+    # TODO replace all entries < 10 with "others"
-    for entry in entries:
+    # TODO make Pie chart out of it
-        if field in entry['Building']:
-            build_systems.extend(entry['Building'][field])
-    build_systems = [x.value for x in build_systems]
-
-    unique_build_systems = set(build_systems)
-    unique_build_systems = [(l, build_systems.count(l)) for l in unique_build_systems]
-    unique_build_systems.sort(key=lambda x: str.casefold(x[0]))  # first sort by name
-    unique_build_systems.sort(key=lambda x: -x[1])  # then sort by occurrence (highest occurrence first)
     section = {
         'title': 'Build system',
-        'items': ['{} ({})'.format(*item) for item in unique_build_systems]
+        'items': ['{} ({})'.format(*item) for item in build_systems_stat]
     }
     data['sections'].append(section)
+
+    # render and write statistics page
     write(template.render(data=data), ['statistics.html'])
 
-    # frameworks folder
+    # non-games folder
-    base['url_to'] = partial(url_to, frameworks_path)
+    base['url_to'] = partial(url_to, non_games_path)
     base['active_nav'] = 'frameworks'
 
-    # frameworks by type
+    # non-games by type
-    index = divide_in_three_columns_and_transform(frameworks_by_type, game_index)
+    index = divide_in_three_columns_and_transform(non_games_by_type, game_index)
     index['title'] = make_text('Open source game frameworks/tools')
-    index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(frameworks)))
+    index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(non_games)))
     index['categories'] = c.non_game_keywords
-    index['category-names'] = framework_names
+    index['category-names'] = non_game_category_names
     index['category-icons'] = {}
     index['number_entries_per_category_threshold'] = 0
     index['entry_bold'] = lambda x: 'tags' not in x
     index['category-infos'] = {}
-    write(template_categorical_index.render(index=index), frameworks_index_path)
+    write(template_categorical_index.render(index=index), non_games_index_path)
 
-    # generate frameworks pages
+    # generate non-games pages
     for keyword in c.non_game_keywords:
         listing = {
-            'title': framework_names[keyword],
+            'title': non_game_category_names[keyword],
-            'subtitle': make_url(frameworks_index_path, 'Index'),
+            'subtitle': make_url(non_games_index_path, 'Index'),
-            'items': frameworks_by_type[keyword]
+            'items': non_games_by_type[keyword]
         }
-        write(template_listing_entries.render(listing=listing), frameworks_path +['{}.html'.format(keyword)])
+        write(template_listing_entries.render(listing=listing), non_games_path + ['{}.html'.format(keyword)])
 
     # games folder
     base['url_to'] = partial(url_to, games_path)

code/html/statistics.jinja

@@ -1,12 +1,13 @@
 {% extends "base.jinja" %}
 {% block content %}
-  <div class="container">
+<div class="container">
-    <p class="title is-4">{{ data['title'] }}</p>
+  <div class="box"><p class="title is-4">{{ data['title'] }}</p></div>
 {% set comma = joiner(",") %}
 {% for section in data['sections'] %}
     {{ comma() }} <a href="#">{{ section['title'] }}</a>
 {% endfor %}
     </div>
+                            {#- each statistics section -#}
 {% for section in data['sections'] %}
   <section class="section">
     <div class="container">

code/maintenance_developers.py

@@ -100,7 +100,6 @@ class DevelopersMaintainer:
             entry_name = entry['Title']
             entry_devs = entry.get('Developer', [])
             for entry_dev in entry_devs:
-                entry_dev = entry_dev.value  # ignore a possible comment
                 if entry_dev in self.developers:
                     self.developers[entry_dev]['Games'].append(entry_name)
                 else:
@@ -127,7 +126,7 @@ class DevelopersMaintainer:
         # for entry in self.entries:
         #     for developer in entry.get('Developer', []):
         #         if developer.comment:
-        #             print('{:<25} - {:<25} - {}'.format(entry['File'], developer.value, developer.comment))
+        #             print('{:<25} - {:<25} - {}'.format(entry['File'], developer, developer.comment))
 
 
 if __name__ == "__main__":

code/maintenance_entries.py

@@ -56,7 +56,6 @@ def create_toc(title, file, entries):
     rows = []
     for entry in entries:
         info = entry['Code language'] + entry['Code license'] + entry['State']
-        info = [x.value for x in info]
         rows.append('- **[{}]({})** ({})'.format(entry['Title'], '../' + entry['File'], ', '.join(info)))
 
     # sort rows (by title)
@@ -130,7 +129,6 @@ class EntriesMaintainer:
             keywords.extend(entry['Keyword'])
             if b'first\xe2\x80\x90person'.decode() in entry['Keyword']:
                 print(entry['File'])
-        keywords = [x.value for x in keywords]
 
         # reduce those starting with "multiplayer"
         keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords]
@@ -159,7 +157,6 @@ class EntriesMaintainer:
         for entry in self.entries:
             deps = entry.get('Code dependency', [])
             for dependency in deps:
-                dependency = dependency.value
                 if dependency in referenced_dependencies:
                     referenced_dependencies[dependency] += 1
                 else:
@@ -516,7 +513,6 @@ class EntriesMaintainer:
         languages = []
         for entry in self.entries:
             languages.extend(entry[field])
-        languages = [x.value for x in languages]
 
         unique_languages = set(languages)
         unique_languages = [(l, languages.count(l) / len(languages)) for l in unique_languages]
@@ -538,7 +534,6 @@ class EntriesMaintainer:
         licenses = []
         for entry in self.entries:
             licenses.extend(entry[field])
-        licenses = [x.value for x in licenses]
 
         unique_licenses = set(licenses)
         unique_licenses = [(l, licenses.count(l) / len(licenses)) for l in unique_licenses]
@@ -560,7 +555,6 @@ class EntriesMaintainer:
         keywords = []
         for entry in self.entries:
             keywords.extend(entry[field])
-        keywords = [x.value for x in keywords]
 
         # reduce those starting with "multiplayer"
         keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords]
@@ -597,7 +591,7 @@ class EntriesMaintainer:
             popular = False
             for repo in entry.get(field, []):
                 for popular_repo in popular_code_repositories:
-                    if popular_repo in repo.value:
+                    if popular_repo in repo:
                         popular = True
                         break
             # if there were repositories, but none popular, add them to the list
@@ -618,7 +612,6 @@ class EntriesMaintainer:
             if field in entry:
                 code_dependencies.extend(entry[field])
                 entries_with_code_dependency += 1
-        code_dependencies = [x.value for x in code_dependencies]
         statistics += 'With code dependency field {} ({:.1f}%)\n\n'.format(entries_with_code_dependency,
                                                                            rel(entries_with_code_dependency))
 
@@ -644,7 +637,6 @@ class EntriesMaintainer:
         for entry in self.entries:
             if field in entry['Building']:
                 build_systems.extend(entry['Building'][field])
-        build_systems = [x.value for x in build_systems]
 
         statistics += 'Build systems information available for {:.1f}% of all projects.\n\n'.format(
             rel(len(build_systems)))
@@ -690,7 +682,6 @@ class EntriesMaintainer:
         for entry in self.entries:
             if field in entry:
                 platforms.extend(entry[field])
-        platforms = [x.value for x in platforms]
 
         statistics += 'Platform information available for {:.1f}% of all projects.\n\n'.format(rel(len(platforms)))
 
@@ -708,67 +699,12 @@ class EntriesMaintainer:
 
     def update_html(self):
         """
-        Parses all entries, collects interesting info and stores it in a json file suitable for displaying
-        with a dynamic table in a browser.
         """
         if not self.entries:
             print('entries not yet loaded')
             return
 
-        # make database out of it
+        print('HTML not updated')
-        db = {'headings': ['Game', 'Description', 'Download', 'State', 'Keyword', 'Source']}
-
-        entries = []
-        for info in self.entries:
-
-            # game & description
-            entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['Title'], info['Home'][0],
-                                                                                 r'https://github.com/Trilarion/opensourcegames/blob/master/entries/' +
-                                                                                 info['File']),
-                     textwrap.shorten(info.get('Note', ''), width=60, placeholder='..')]
-
-            # download
-            field = 'Download'
-            if field in info and info[field]:
-                entry.append('<a href="{}">Link</a>'.format(info[field][0]))
-            else:
-                entry.append('')
-
-            # state (field state is essential)
-            entry.append('{} / {}'.format(info['State'][0],
-                                          'inactive since {}'.format(osg.extract_inactive_year(info)) if osg.is_inactive(info) else 'active'))
-
-            # keywords
-            keywords = info['Keyword']
-            keywords = [x.value for x in keywords]
-            entry.append(', '.join(keywords))
-
-            # source
-            text = []
-            field = 'Code repository'
-            if field in info and info[field]:
-                text.append('<a href="{}">Source</a>'.format(info[field][0].value))
-            languages = info['Code language']
-            languages = [x.value for x in languages]
-            text.append(', '.join(languages))
-            licenses = info['Code license']
-            licenses = [x.value for x in licenses]
-            text.append(', '.join(licenses))
-            entry.append(' - '.join(text))
-
-            # append to entries
-            entries.append(entry)
-
-        # sort entries by game name
-        entries.sort(key=lambda x: str.casefold(x[0]))
-
-        db['data'] = entries
-
-        # output
-        text = json.dumps(db, indent=1)
-        utils.write_text(c.json_db_file, text)
-
-        print('HTML updated')
 
     def update_repos(self):
         """
@@ -784,7 +720,6 @@ class EntriesMaintainer:
         # for every entry filter those that are known git repositories (add additional repositories)
         for entry in self.entries:
             repos = entry.get('Code repository', [])
-            repos = [x.value for x in repos]
             # keep the first and all others containing @add
             if not repos:
                 continue
@@ -841,7 +776,6 @@ class EntriesMaintainer:
         git_repos = []
         for entry in self.entries:
             repos = entry['Code repository']
-            repos = [x.value for x in repos]
             for repo in repos:
                 repo = repo.split(' ')[0].strip()
                 url = osg.git_repo(repo)
@@ -898,7 +832,7 @@ class EntriesMaintainer:
         # stats = {}
         # for entry in self.entries:
         #     repos = entry.get('Code repository', [])
-        #     comments = [x.comment for x in repos if x.value.startswith('https://github.com/') and x.comment]
+        #     comments = [x.comment for x in repos if x.startswith('https://github.com/') and x.comment]
         #     for comment in comments:
         #         for part in comment.split(','):
         #             part = part.strip()

code/maintenance_inspirations.py

@@ -152,7 +152,6 @@ class InspirationMaintainer:
         for entry in self.entries:
             entry_name = entry['Title']
             for inspiration in entry.get('Inspiration', []):
-                inspiration = inspiration.value
                 if inspiration in self.inspirations:
                     self.inspirations[inspiration]['Inspired entries'].append(entry_name)
                 else:

code/rejected.txt

@@ -79,6 +79,7 @@ HistoryLine (https://github.com/oliverdb/Historyline): Very early development, n
 Homeworld SDL (https://github.com/aheadley/homeworld): Not open source compliant license (see https://github.com/aheadley/homeworld/blob/master/README)
 Howitzer Skirmish (http://howski.sourceforge.net/): No source code
 Ikariam (https://github.com/advocaite/ikariam, https://github.com/TheOnly92/Ikariem): No license information found, no assets license information found
+Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files)
 imperialism-remake (http://remake.twelvepm.de/, https://github.com/Trilarion/imperialism-Remake): Too minimal (I must know)
 Imperium: Sticks (http://rtciv.sourceforge.net/, http://sourceforge.net/projects/rtciv): No source code available
 ImperiumAO (https://sourceforge.net/projects/impao, https://www.imperiumao.com.ar/): Only engine is open source, engine is ORE
@@ -209,5 +210,4 @@ XQuest 2 (http://www.swallowtail.org/xquest/, http://www.swallowtail.org/xquest/
 xrick (http://www.bigorno.net/xrick): No open source license/unclear license (see file README in http://www.bigorno.net/xrick/xrick-021212.zip)
 Yave (https://github.com/gan74/Yave): General graphics engine, not game centered in any way and experimental
 Yuris Revenge (https://github.com/cookgreen/Yuris-Revenge): Mod to OpenRA
-zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance
+zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance
-Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files)

code/synchronization/osgameclones_synchronization.py

@@ -281,14 +281,14 @@ if __name__ == "__main__":
                     osgc_languages = osgc_entry['lang']
                     if type(osgc_languages) == str:
                         osgc_languages = [osgc_languages]
-                    our_languages = [x.value for x in our_entry['Code language']]  # essential field
+                    our_languages = our_entry['Code language']
                     p += compare_sets(osgc_languages, our_languages, 'code language')
 
                 # compare their license with our code and assets license
                 if 'license' in osgc_entry:
                     osgc_licenses = osgc_entry['license']
-                    our_code_licenses = [x.value for x in our_entry['Code license']]  # essential field
+                    our_code_licenses = our_entry['Code license']  # essential field
-                    our_assets_licenses = [x.value for x in our_entry.get('Assets license', [])]
+                    our_assets_licenses = our_entry.get('Assets license', [])
                     p += compare_sets(osgc_licenses, our_code_licenses + our_assets_licenses, 'licenses', 'notthem')
                     p += compare_sets(osgc_licenses, our_code_licenses, 'licenses', 'notus')
 
@@ -298,7 +298,7 @@ if __name__ == "__main__":
                     osgc_frameworks = osgc_entry['framework']
                     if type(osgc_frameworks) == str:
                         osgc_frameworks = [osgc_frameworks]
-                    our_frameworks = [x.value for x in our_entry.get('Code dependency', [])]
+                    our_frameworks = our_entry.get('Code dependency', [])
                     our_frameworks = [x.casefold() for x in our_frameworks]
                     our_frameworks = [x if x not in our_framework_replacements else our_framework_replacements[x] for x
                                       in our_frameworks]
@@ -315,13 +315,13 @@ if __name__ == "__main__":
                         'sourceforge.net/projects/')]  # we don't need the general sites there
                     # osgc_repos = [x for x in osgc_repos if not x.startswith('https://sourceforge.net/projects/')] # ignore some
                     our_repos = our_entry.get('Code repository', [])
-                    our_repos = [utils.strip_url(url.value) for url in our_repos]
+                    our_repos = [utils.strip_url(url) for url in our_repos]
                     our_repos = [x for x in our_repos if not x.startswith(
                         'gitlab.com/osgames/')]  # we do not yet spread our own deeds (but we will some day)
                     our_repos = [x for x in our_repos if
                                  'cvs.sourceforge.net' not in x and 'svn.code.sf.net/p/' not in x]  # no cvs or svn anymore
                     our_downloads = our_entry.get('Download', [])
-                    our_downloads = [utils.strip_url(url.value) for url in our_downloads]
+                    our_downloads = [utils.strip_url(url) for url in our_downloads]
                     p += compare_sets(osgc_repos, our_repos + our_downloads, 'repo',
                                       'notthem')  # if their repos are not in our downloads or repos
                     p += compare_sets(osgc_repos, our_repos[:1], 'repo',
@@ -334,7 +334,7 @@ if __name__ == "__main__":
                         osgc_urls = [osgc_urls]
                     osgc_urls = [utils.strip_url(url) for url in osgc_urls]
                     our_urls = our_entry['Home']
-                    our_urls = [utils.strip_url(url.value) for url in our_urls]
+                    our_urls = [utils.strip_url(url) for url in our_urls]
                     p += compare_sets(osgc_urls, our_urls, 'url/home', 'notthem')  # if their urls are not in our urls
                     # our_urls = [url for url in our_urls if
                     #             not url.startswith('github.com/')]  # they don't have them as url
@@ -361,10 +361,10 @@ if __name__ == "__main__":
                         p += ' development : mismatch : them complete, us not mature\n'
 
                 # get our keywords
-                our_keywords = [x.value for x in our_entry['Keyword']] # essential
+                our_keywords = our_entry['Keyword'] # essential
 
                 # compare their originals to our inspirations
-                our_originals = [x.value for x in our_entry.get('Inspiration', [])]
+                our_originals = our_entry.get('Inspiration', [])
                 if 'originals' in osgc_entry:
                     osgc_originals = osgc_entry['originals']
                     osgc_originals = [x.replace(',', '') for x in

code/synchronization/sourceforge_import.py

@@ -76,7 +76,7 @@ def sourceforge_import():
             # read full entry
             entry = osg.read_entry(file)
             developers = entry.get('Developer', [])
-            urls = [x.value for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')]
+            urls = [x for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')]
 
             # do we need to save it again
             entry_changed = False
@@ -133,7 +133,7 @@ def sourceforge_import():
                     # look author up in entry developers field, if not existing add
                     if author_name not in developers:
                         print('   dev "{}" added to entry {}'.format(author_name, file))
-                        entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(author_name)]
+                        entry['Developer'] = entry.get('Developer', []) + [author_name]
                         entry_changed = True
                         developers = entry.get('Developer', [])  # update developers
 

code/utils/constants.py

@@ -68,9 +68,10 @@ fields_without_comments = ('Inspiration', 'Play', 'Download', 'Platform', 'Code
 recommended_keywords = (
     'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation',
     'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
+# TODO unmake remake a recommended keyword (should be the same as clone maybe), i.e. add another recommended keyword if only remake is in there
 
-# non game keywords take precedence
+# non game keywords take precedence over other (game) recommended keywords, at most one of them per entry
-non_game_keywords = ('framework', 'library', 'tool')
+non_game_keywords = ('framework', 'game engine', 'library', 'tool')
 
 # known programming languages, anything else will result in a warning during a maintenance operation
 # only these will be used when gathering statistics

code/utils/osg.py

@@ -340,11 +340,11 @@ def check_and_process_entry(entry):
     if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
         message += 'file name should be {}\n'.format(canonical_file_name)
 
-    # check that fields without comments have no comments, set to field without comment
+    # check that fields without comments have no comments (i.e. are no Values)
     for field in c.fields_without_comments:
         if field in entry:
             content = entry[field]
-            if any(item.has_comment() for item in content):
+            if any(isinstance(item, osg_parse.Value) for item in content):
                 message += 'field without comments {} has comment\n'.format(field)
 
     # state must contain either beta or mature but not both
@@ -359,8 +359,8 @@ def check_and_process_entry(entry):
     for field in c.url_fields:
         values = entry.get(field, [])
         for value in values:
-            if value.value.startswith('<') and value.value.endswith('>'):
+            if value.startswith('<') and value.endswith('>'):
-                value.value = value.value[1:-1]
+                value = value[1:-1]
             if not any(value.startswith(x) for x in c.valid_url_prefixes):
                 message += 'URL "{}" in field "{}" does not start with a valid prefix'.format(value, field)
 
@@ -368,7 +368,7 @@ def check_and_process_entry(entry):
     for repo in entry.get('Code repository', []):
         if any(repo.startswith(x) for x in ('@', '?')):
             continue
-        repo = repo.value.split(' ')[0].strip()
+        repo = repo.split(' ')[0].strip()
         if any((x in repo for x in ('github', 'gitlab', 'git.tuxfamily', 'git.savannah'))):
                 if not repo.startswith('https://'):
                     message += 'Repo "{}" should start with https://'.format(repo)
@@ -420,7 +420,7 @@ def is_inactive(entry):
 def extract_inactive_year(entry):
     state = entry['State']
     phrase = 'inactive since '
-    inactive_year = [x.value[len(phrase):] for x in state if x.startswith(phrase)]
+    inactive_year = [x[len(phrase):] for x in state if x.startswith(phrase)]
     assert len(inactive_year) <= 1
     if inactive_year:
         return int(inactive_year[0])
@@ -457,9 +457,29 @@ def write_entry(entry):
     utils.write_text(entry_path, content)
 
 
-def create_entry_content(entry):
+def render_value(value):
     """
 
+    :param value:
+    :return:
+    """
+    if isinstance(value, osg_parse.Value):
+        comment = value.comment
+    else:
+        comment = None
+    if any(x in value for x in (',', ' (')):
+        value = '"{}"'.format(value)
+    if comment:
+        return '{} ({})'.format(value, comment)
+    else:
+        return value
+
+
+def create_entry_content(entry):
+    """
+    Creates the entry content from an internal representation as dictionary with fields to a text file representation
+    that can be stored in the md files. It should be compatible with the gramar and reading a file and re-creating the
+    content should not change the content. Importanly, the comments of the values have to be added here.
     :param entry:
     :return:
     """
@@ -468,7 +488,7 @@ def create_entry_content(entry):
     content = '# {}\n\n'.format(entry['Title'])
 
     # we automatically sort some fields
-    sort_fun = lambda x: str.casefold(x.value)
+    sort_fun = lambda x: str.casefold(x)
     for field in ('Media', 'Inspiration', 'Code Language', 'Developer', 'Build system'):
         if field in entry:
             values = entry[field]
@@ -479,12 +499,11 @@ def create_entry_content(entry):
     b = [x for x in keywords if x not in c.recommended_keywords]
     entry['Keyword'] = sorted(a, key=sort_fun) + sorted(b, key=sort_fun)
 
-    # now all properties in the recommended order
+    # now all properties are in the recommended order
     for field in c.valid_properties:
         if field in entry:
             e = entry[field]
-            e = ['"{}"'.format(x) if any(y in x.value for y in (',', ' (')) else x for x in e]
+            e = [render_value(x) for x in e]
-            e = [str(x) for x in e]
             e = list(dict.fromkeys(e))  # this removes duplicates while keeping the sorting order
             content += '- {}: {}\n'.format(field, ', '.join(e))
     content += '\n'
@@ -504,8 +523,8 @@ def create_entry_content(entry):
                 has_properties = True
                 content += '\n'
             e = entry['Building'][field]
-            e = ['"{}"'.format(x) if ',' in x else x for x in e]
+            e = [render_value(x) for x in e]
-            e = [str(x) for x in e]
+            e = list(dict.fromkeys(e))  # this removes duplicates while keeping the sorting order
             content += '- {}: {}\n'.format(field, ', '.join(e))
 
     # if there is a note, insert it
@@ -533,16 +552,14 @@ def all_urls(entries):
     :param entries: 
     :return: 
     """
+    # TODO there are other fields than c.url_fields and also in comments, maybe just regex on the whole content
+    # TODO this might be part of the external link check or it might not, check for duplicate code
     urls = {}
     # iterate over entries
     for entry in entries:
         file = entry['File']
-        for field in c.url_fields:  # TODO there are other fields, maybe just regex on the whole content
+        for field in c.url_fields:
             for value in entry.get(field, []):
-                if value.comment:
-                    value = value.value + ' ' + value.comment
-                else:
-                    value = value.value
                 for subvalue in value.split(' '):
                     subvalue = subvalue.strip()
                     if is_url(subvalue):

code/utils/osg_parse.py

@@ -14,10 +14,10 @@ class ListingTransformer(lark.Transformer):
     """
 
     def unquoted_value(self, x):
-        return x[0].value.strip()
+        return x[0].strip()
 
     def quoted_value(self, x):
-        return x[0].value[1:-1].strip()  # remove quotation marks and strip whitespaces
+        return x[0][1:-1].strip()  # remove quotation marks and strip whitespaces
 
     def property(self, x):
         """
@@ -25,7 +25,7 @@ class ListingTransformer(lark.Transformer):
         :param x:
         :return:
         """
-        return x[0].value, x[1:]
+        return x[0], x[1:]
 
     def name(self, x):
         """
@@ -33,7 +33,7 @@ class ListingTransformer(lark.Transformer):
         :param x:
         :return:
         """
-        return 'Name', x[0].value.strip()
+        return 'Name', x[0].strip()
 
     def entry(self, x):
         """
@@ -56,19 +56,25 @@ class ListingTransformer(lark.Transformer):
 class EntryTransformer(lark.Transformer):
 
     def unquoted_value(self, x):
-        return x[0].value.strip()
+        return x[0].strip()
 
     def quoted_value(self, x):
-        return x[0].value[1:-1].strip()  # remove quotation marks
+        return x[0][1:-1].strip()  # remove quotation marks
 
     def comment_value(self, x):
-        return x[0].value[1:-1].strip()  # remove parenthesis
+        return x[0][1:-1].strip()  # remove parenthesis
 
     def value(self, x):
+        """
+        This also stores the comment if needed.
+
+        :param x:
+        :return:
+        """
         if len(x) == 1:
-            v = ValueWithComment(value=x[0])
+            v = x[0]
         else:
-            v = ValueWithComment(value=x[0], comment=x[1])
+            v = Value(*x)
         return v
 
     def property(self, x):
@@ -77,10 +83,10 @@ class EntryTransformer(lark.Transformer):
         :param x:
         :return:
         """
-        return x[0].value.strip(), x[1:]
+        return x[0].strip(), x[1:]
 
     def title(self, x):
-        return 'Title', x[0].value.strip()
+        return 'Title', x[0].strip()
 
     def note(self, x):
         """
@@ -90,7 +96,7 @@ class EntryTransformer(lark.Transformer):
         """
         if not x:
             raise lark.Discard
-        return 'Note', ''.join((x.value for x in x))
+        return 'Note', ''.join(x)
 
     def building(self, x):
         return 'Building', x
@@ -98,40 +104,16 @@ class EntryTransformer(lark.Transformer):
     def start(self, x):
         return x
 
-# TODO turns out ValueWithComment does not really solve problem but actually creates even some, are there alternatives like inheriting from string?
+
-class ValueWithComment:
+class Value(str):
     """
-      All our property values can have (optional) comments. This is the class that represents them to us and implements
+    A value is a string with some additional meta object (a comment) but mostly behaves as a string.
-      equality and 'in' operator functionality purely on the value.
     """
 
-    def __init__(self, value, comment=None):
+    def __new__(cls, value, comment):
-        self.value = value
+        obj = str.__new__(cls, value)
-        self.comment = comment
+        obj.comment = comment
-
+        return obj
-    def is_empty(self):
-        return self.value == ''
-
-    def has_comment(self):
-        return self.comment is not None
-
-    def startswith(self, str):
-        return self.value.startswith(str)
-
-    def __contains__(self, item):
-        return item in self.value
-
-    def __eq__(self, other):
-        return self.value == other
-
-    def __repr__(self):
-        if self.comment:
-            return '{} ({})'.format(self.value, self.comment)
-        else:
-            return '{}'.format(self.value)
-
-    def __hash__(self):
-        return hash(self.value)
 
 def parse(parser, transformer, content):
     tree = parser.parse(content)

code/utils/osg_statistics.py

@@ -1,4 +1,23 @@
 """
+Central place to calculate statistics about the entries. Used for updating the statistics.md file and the statistics page
+of the website.
+"""
 
-"""
 
+def get_build_systems(entries):
+    """
+    Given a list of entries, calculates statistics about the used build systems and returns the statistics as
+    sorted list of elements (build-system-name, occurence).
+    "n/a" is used if no build system was specified
+    """
+    build_systems = []
+    for entry in entries:
+        build_systems.extend(entry['Building'].get('Build system', ['n/a']))
+
+    unique_build_systems = set(build_systems)
+
+    build_systems_stat = [(l, build_systems.count(l)) for l in unique_build_systems]
+    build_systems_stat.sort(key=lambda x: str.casefold(x[0]))  # first sort by name
+    build_systems_stat.sort(key=lambda x: -x[1])  # then sort by occurrence (highest occurrence first)
+
+    return build_systems_stat

entries/freee.md

@@ -2,9 +2,9 @@
 
 - Home: http://edkolis.com/freee
 - Inspiration: Space Empires IV
-- State: beta 
+- State: beta
 - Download: https://github.com/ekolis/FrEee/releases
-- Keyword: strategy, turn-based, space, content open
+- Keyword: strategy, content open, space, turn-based
 - Code repository: https://github.com/ekolis/FrEee.git
 - Code language: C#
 - Code license: CC-BY-NC-SA-2.0 (https://github.com/ekolis/FrEee/blob/master/FrEee.Assets/Licenses/license.txt)
@@ -12,4 +12,4 @@
 
 ## Building
 
-- Build system: Visual Studio
+- Build system: Visual Studio