screenshot (github top 50) and statistics charts support in website output

game engines now counted as frameworks
few additions

code/backlog.txt

@@ -98,4 +98,5 @@ https://www.seul.org/~grumbel/tmp/clanlib/games.html
 https://www.tapatalk.com/groups/imperilist/
 https://www.wurfelengine.net/
 https://zdoom.org/downloads (gzdoom, lzdoom)
-https://zope.readthedocs.io/en/latest/
+https://zope.readthedocs.io/en/latest/
+https://github.com/loudsmilestudios/TetraForce

code/html/base.jinja

@@ -38,7 +38,7 @@
       <a class="navbar-item{% if 'frameworks' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['frameworks', 'index.html']) }}">{{ macros.render_icon({'class':'wrench'}) }}<span>Frameworks/Tools</span></a>
       <a class="navbar-item{% if 'developers' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['developers', 'index.html']) }}">{{ macros.render_icon({'class':'users'}) }}<span>Developers</span></a>
       <a class="navbar-item{% if 'inspirations' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['inspirations', 'index.html']) }}">{{ macros.render_icon({'class':'bulb'}) }}<span>Inspirations</span></a>
-      <a class="navbar-item{% if 'statistics' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['statistics.html']) }}">{{ macros.render_icon({'class':'stats-dots'}) }}<span>Statistics</span></a>
+      <a class="navbar-item{% if 'statistics' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['statistics', 'index.html']) }}">{{ macros.render_icon({'class':'stats-dots'}) }}<span>Statistics</span></a>
       <a class="navbar-item{% if 'contribute' in base['active_nav'] %} is-active{% endif %}" href="{{ base['url_to'](['contribute.html']) }}">{{ macros.render_icon({'class':'pencil'}) }}<span>Contribute</span></a>
       <a class="navbar-item" href="https://github.com/Trilarion/opensourcegames">{{ macros.render_icon({'class':'github'}) }}<span>On GitHub</span></a>
     </div>

code/html/generate_static_website.py

@@ -10,6 +10,12 @@ Sitemaps is not needed, only for large projects with lots of JavaScript und many
 # TODO most people only come to the main page, put more information there (direct links to genres, ...)
 # TODO put more explanations on the category pages and the categories (number and short sentences)
 
+# TODO text description automaticall generated from keywords, state, and technical informations for example: First-person action shooter written in C++, inspired by ... but inactive for 7 years.
+
+# TODO statistics should have disclaimer (warning or info box) about accuracy with link to contribute guidelines
+
+# TODO game move developer to technical information and maybe rename technical information to details
+
 # TODO game engines should be sorted with frameworks/tools, not with games (they aren't games or are they?)
 
 # TODO if the only change is a change in last updated, do not change it (we can probably check with git diff for it) or checksums
@@ -23,6 +29,8 @@ Sitemaps is not needed, only for large projects with lots of JavaScript und many
 
 # TODO everywhere: singular, plural (game, entries, items)
 
+# TODO games: platform icons and mature, state larger (but maybe not on mobile)
+
 # TODO statistics: better and more statistics with links where possible
 # TODO statistics: with nice graphics (pie charts in SVG) with matplotlib
 # TODO statistics: get it from common statistics generator
@@ -74,6 +82,8 @@ Sitemaps is not needed, only for large projects with lots of JavaScript und many
 
 # TODO add dynamic table (what to include)
 
+# TODO meta titles for all pages, make them nice because they appear in search results! (https://www.contentpowered.com/blog/good-ctr-search-console/)
+
 import os
 import shutil
 import math
@@ -100,12 +110,14 @@ games_path = ['games']
 non_games_path = ['frameworks']
 inspirations_path = ['inspirations']
 developers_path = ['developers']
+statistics_path = ['statistics']
 
 # derived paths
 games_index_path = games_path + ['index.html']
 non_games_index_path = non_games_path + ['index.html']
 inspirations_index_path = inspirations_path + ['index.html']
 developers_index_path = developers_path + ['index.html']
+statistics_index_path = statistics_path + ['index.html']
 
 games_by_language_path = games_path + ['languages.html']
 games_by_genres_path = games_path + ['genres.html']
@@ -548,6 +560,20 @@ def make_tags(entries):
     }
 
 
+def make_detail(summary, detail):
+    """
+
+    :param summary:
+    :param detail:
+    :return:
+    """
+    return {
+        'type': 'detail',
+        'summary': summary,
+        'detail': detail
+    }
+
+
 def developer_profile_link(link):
     """
     Creates links to developer profiles.
@@ -637,10 +663,12 @@ def create_keyword_tag(keyword):
         else:
             url = games_by_genres_path.copy()
         url[-1] += '#{}'.format(keyword)
-        if keyword.capitalize() in genre_icon_map:
-            return make_url(url, [make_icon(genre_icon_map[keyword.capitalize()]), make_text(keyword)], '{} games'.format(keyword), 'tag is-info')
-        else:
-            return make_url(url, make_text(keyword), '{} games'.format(keyword), 'tag is-info')
+        # TODO are icons looking good in the keyword tags (I somehow doubt it), maybe put them separately somewhere?
+        #if keyword.capitalize() in genre_icon_map:
+        #    return make_url(url, [make_icon(genre_icon_map[keyword.capitalize()]), make_text(keyword)], '{} games'.format(keyword), 'tag is-info')
+        #else:
+        #    return make_url(url, make_text(keyword), '{} games'.format(keyword), 'tag is-info')
+        return make_url(url, make_text(keyword), '{} games'.format(keyword), 'tag is-info')
     else:
         return make_text(keyword, 'tag is-light')
 
@@ -698,10 +726,7 @@ def convert_entries(entries, inspirations, developers):
                 if field == 'Inspiration':
                     e = [make_url(inspirations_references[x], make_text(x)) for x in e]
                 elif field == 'Developer':
-                    if len(e) > 10: # many devs, print smaller
-                        e = [make_url(developer_references[x], make_text(x, 'is-size-7')) for x in e]
-                    else:
-                        e = [make_url(developer_references[x], make_text(x)) for x in e]
+                    e = [make_url(developer_references[x], make_text(x)) for x in e]
                 elif field in c.url_fields:
                     e = [make_url(x, shortcut_url(x, name)) for x in e]
                 else:
@@ -777,7 +802,7 @@ def get_top50_games(games):
     for game in games:
         # get stars of repositories
         stars = 0
-        for repo in game.get('Code repository', []):
+        for repo in game.get('Code repository', [])[:1]:  # take at most one
             if isinstance(repo, osg_parse.Value):
                 for c in repo.comment.split(','):
                     c = c.strip()
@@ -798,6 +823,25 @@ def get_top50_games(games):
     return top50_games
 
 
+def add_screenshot_information(entries):
+    """
+
+    :param entries:
+    :return:
+    """
+    # read screenshot information
+    overview = osg.read_screenshots_overview()
+
+    # iterate over entries
+    for entry in entries:
+        # get screenshots entry
+        name = osg.canonical_name(entry['Title'])  # TODO should be stored upon loading I guess
+        screenshots = overview.get(name, {})
+        screenshots = [{'width': s[0], 'height': s[1], 'url': s[2], 'file': ['screenshots', '{}_{:02d}.jpg'.format(name, id)]} for id, s in screenshots.items()]
+        if screenshots: # TODO url None should be treated here or in the jinja template
+            entry['screenshots'] = screenshots
+
+
 def generate(entries, inspirations, developers):
     """
     Regenerates the whole static website given an already imported set of entries, inspirations and developers.
@@ -856,6 +900,12 @@ def generate(entries, inspirations, developers):
     utils.copy_tree(os.path.join(c.web_template_path, 'css'), c.web_css_path)
     utils.copy_tree(os.path.join(c.web_template_path, 'js'), c.web_js_path)
 
+    # copy screenshots path
+    files = [file for file in os.listdir(c.screenshots_path) if file.endswith('.jpg')]
+    os.makedirs(c.web_screenshots_path, exist_ok=True)
+    for file in files:
+        shutil.copyfile(os.path.join(c.screenshots_path, file), os.path.join(c.web_screenshots_path, file))
+
     # collage_image and google search console token and favicon.svg
     for file in ('collage_games.jpg', 'google1f8a3863114cbcb3.html', 'favicon.svg'):
         shutil.copyfile(os.path.join(c.web_template_path, file), os.path.join(c.web_path, file))
@@ -884,7 +934,8 @@ def generate(entries, inspirations, developers):
     template = environment.get_template('contribute.jinja')
     write(template.render(), ['contribute.html'])
 
-    # statistics page
+    # statistics page in statistics folder
+    base['url_to'] = partial(url_to, statistics_path)
     base['active_nav'] = 'statistics'
 
     # statistics preparation
@@ -896,16 +947,17 @@ def generate(entries, inspirations, developers):
 
     # build-systems
     build_systems_stat = stat.get_build_systems(entries)
-    # TODO replace all entries < 10 with "others"
-    # TODO make Pie chart out of it
+    build_systems_stat = stat.truncate_stats(build_systems_stat, 10)
+    stat.export_pie_chart([stat for stat in build_systems_stat if stat[0] != 'N/A'], os.path.join(c.web_path, 'statistics', 'build_systems.svg'))
     section = {
         'title': 'Build system',
-        'items': ['{} ({})'.format(*item) for item in build_systems_stat]
+        'items': ['{} ({})'.format(*item) for item in build_systems_stat],
+        'chart': statistics_path + ['build_systems.svg']
     }
     data['sections'].append(section)
 
     # render and write statistics page
-    write(template.render(data=data), ['statistics.html'])
+    write(template.render(data=data), statistics_index_path)
 
     # non-games folder
     base['url_to'] = partial(url_to, non_games_path)
@@ -998,6 +1050,9 @@ def generate(entries, inspirations, developers):
 
     # top 50 games
     base['active_nav'] = ['filter', 'top50']
+    # there are no other games coming afterwards, can actually number them
+    for index, game in enumerate(top50_games):
+        game['name'] = '{}. '.format(index+1) + game['name']
     listing = {
         'title': 'GitHub Stars Top 50',
         'subtitle': '50 highest rated (by stars on Github) open source games in the database',
@@ -1066,11 +1121,14 @@ if __name__ == "__main__":
     print('clean current static website')
     utils.recreate_directory(c.web_path)
 
-    # load entries, inspirations and developers and sort them
+    # load entries, inspirations and developers and sort them alphabetically
     print('load entries, inspirations and developers')
     entries = osg.read_entries()
     entries.sort(key=lambda x: str.casefold(x['Title']))
 
+    # add screenshot information
+    add_screenshot_information(entries)
+
     inspirations = osg.read_inspirations()
     inspirations = list(inspirations.values())
     inspirations.sort(key=lambda x: str.casefold(x['Name']))

code/html/listing_entries.jinja

@@ -23,15 +23,27 @@
   </div>
                     {#- important fields in a certain order -#}
   <div class="block">
-  {%- for field in ('homepage', 'media', 'inspiration', 'download', 'play online', 'developer') -%}
+  {%- for field in ('homepage', 'media', 'inspiration', 'download', 'play online') -%}
     {%- if field in item -%}{{ macros.render_element(item[field]) }}<br>{%- endif -%}
   {%- endfor -%}
   </div>
+                    {#- screenshots if available -#}
+  {%- if 'screenshots' in item%}<nav class="level">
+  {%- for screenshot in item['screenshots'] -%}
+    <div class="level-item"><a href="{{ screenshot['url'] }}"><img src="{{ base['url_to'](screenshot['file']) }}" width="{{ screenshot['width'] }}" height="{{ screenshot['height'] }}" alt=""></a></div>
+  {%- endfor -%}
+  </nav>{% endif -%}
                     {#- technical fields -#}
   <div class="block is-size-7">
-    <span class="has-text-weight-semibold">Technical information</span><br>
-    {%- for field in ('code language', 'code license', 'code repository', 'code dependency', 'assets license', 'build system') -%}
-    {%- if field in item -%}{{ macros.render_element(item[field]) }}<br>{%- endif -%}
+    <span class="has-text-weight-semibold">Details</span><br>
+    {%- for field in ('code language', 'code license', 'code repository', 'code dependency', 'assets license', 'build system', 'developer') -%}
+    {%- if field in item -%}
+      {%- if item[field][1]['entries']|length > 10 -%}
+      <details><summary>{{ macros.render_element(item[field][0]) }} ({{ item[field][1]['entries']|length }})</summary>{{ macros.render_element(item[field][1]) }}</details>
+      {%- else -%}
+      {{ macros.render_element(item[field]) }}
+      {%- endif -%}
+    <br>{%- endif -%}
     {%- endfor -%}
   </div>
                     {#- improve, raw -#}

code/html/osgameclones_download_images_create_collage.py

@@ -55,6 +55,7 @@ def download_images():
             except:
                 pass
 
+
 def downsize_images():
     scale_factor = 10
     for file in os.listdir(download_path):

code/html/statistics.jinja

@@ -12,6 +12,7 @@
   <section class="section">
     <div class="container">
     <p id="" class="title is-4">{{ section['title'] }}</p>
+    {% if 'chart' in section %}<img src="{{ base['url_to'](section['chart']) }}">{% endif %}
     <ul>
     {% for item in section['items'] -%}
     <li>{{ item }}</li>

code/maintenance_screenshots.py

@@ -0,0 +1,56 @@
+"""
+
+"""
+import os
+from PIL import Image
+from utils import constants as c, utils as u, osg
+
+HEIGHT = 128
+
+if __name__ == "__main__":
+
+    # read available screenhots (and get widths and heights)
+    files = {}
+    for file in os.listdir(c.screenshots_path):
+        path = os.path.join(c.screenshots_path, file)
+        if os.path.isdir(path) or path == c.screenshots_file:
+            continue
+        if not file.endswith('.jpg'):
+            print('Screenshot with unexpected extension: {}'.format(file))
+        # read with pillow and get width and height
+        with Image.open(path)as im:
+            sz = [im.width, im.height]
+            if sz[1] != HEIGHT:
+                print('Screenshot with unexpected height: {} {}'.format(file, sz[1]))
+        # parse file name (name_xx.jpg)
+        name = file[:-7]
+        id = int(file[-6:-4])
+        if name not in files:
+            files[name] = {}
+        files[name][id] = sz
+
+    # read overview
+    overview = osg.read_screenshots_overview()
+
+    # compare both
+
+    # same keys
+    a = set(files.keys())
+    b = set(overview.keys())
+    ab = a - b
+    ba = b - a
+    if ab:
+        print('Names in screenshot files but not in overview: {}'.format(ab))
+    if ba:
+        print('Names in overview but not in screenshot files: {}'.format(ba))
+
+    # update screenshots overview
+    for name, a in overview.items():  # iterate over overview items
+        b = files[name]  # get corresponding file information
+        for id, ai in a.items():  # iterate over overview screenshots
+            bi = b[id]  # get corresponding file information
+            ai[0] = bi[0]  # update width and height
+            ai[1] = bi[1]
+
+    # update screenshots overview
+    osg.write_screenshots_overview(overview)

code/utils/constants.py

@@ -11,9 +11,11 @@ entries_path = os.path.join(root_path, 'entries')
 tocs_path = os.path.join(entries_path, 'tocs')
 code_path = os.path.join(root_path, 'code')
 web_path = os.path.join(root_path, 'docs')
+screenshots_path = os.path.join(entries_path, 'screenshots')
 web_template_path = os.path.join(code_path, 'html')
 web_css_path = os.path.join(web_path, 'css')
 web_js_path = os.path.join(web_path, 'js')
+web_screenshots_path = os.path.join(web_path, 'screenshots')
 
 private_properties_file = os.path.join(root_path, 'private.properties')
 inspirations_file = os.path.join(root_path, 'inspirations.md')
@@ -22,6 +24,7 @@ developer_file = os.path.join(root_path, 'developers.md')
 backlog_file = os.path.join(code_path, 'backlog.txt')
 rejected_file = os.path.join(code_path, 'rejected.txt')
 statistics_file = os.path.join(root_path, 'statistics.md')
+screenshots_file = os.path.join(screenshots_path, 'README.md')
 json_db_file = os.path.join(root_path, 'docs', 'data.json')
 
 # local config

code/utils/osg.py

@@ -615,4 +615,55 @@ def hg_repo(repo):
         return repo
 
     # not hg
-    return None
+    return None
+
+
+def read_screenshots_overview():
+    """
+
+    :return:
+    """
+    # read screenshots readme and parse
+    overview = {}
+    text = utils.read_text(c.screenshots_file)
+    for entry in text.split('\n# ')[1:]:  # skip first paragraph
+        lines = entry.split('\n')  # split into lines
+        name = lines[0]
+        if name not in overview:
+            overview[name] = {}
+        lines = [line for line in lines[1:] if line]  # include only non-empty lines
+        # for every screenshot
+        for line in lines:
+            values = line.split(' ')  # split into values
+            values = [value for value in values if value]
+            id = int(values[0])  # id (must be there)
+            width = int(values[1])  # width can be 0, will be updated
+            height = int(values[2])  # height can be 0, will be updated
+            if len(values) > 3:  # optional an url
+                url = values[3]
+            else:
+                url = None
+            overview[name][id] = [width, height, url]
+    return overview
+
+
+def write_screenshots_overview(overview):
+    """
+
+    :param overview:
+    :return:
+    """
+    # get preamble
+    text = utils.read_text(c.screenshots_file)
+    text = text.split('\n# ')[0] + '\n'
+
+    for name, a in overview.items():
+        t = '# {}\n\n'.format(name)
+        for id, ai in a.items():
+            if ai[-1] is None:
+                ai = ai[:-1]
+            t += ' '.join(['{:02d}'.format(id)] + [str(x) for x in ai]) + '\n'
+        t += '\n'
+        text += t
+
+    utils.write_text(c.screenshots_file, text)

code/utils/osg_statistics.py

@@ -3,6 +3,9 @@ Central place to calculate statistics about the entries. Used for updating the s
 of the website.
 """
 
+import os
+import matplotlib.pyplot as plt
+
 
 def get_build_systems(entries):
     """
@@ -12,7 +15,7 @@ def get_build_systems(entries):
     """
     build_systems = []
     for entry in entries:
-        build_systems.extend(entry['Building'].get('Build system', ['n/a']))
+        build_systems.extend(entry['Building'].get('Build system', ['N/A']))
 
     unique_build_systems = set(build_systems)
 
@@ -21,3 +24,35 @@ def get_build_systems(entries):
     build_systems_stat.sort(key=lambda x: -x[1])  # then sort by occurrence (highest occurrence first)
 
     return build_systems_stat
+
+
+def truncate_stats(stat, threshold, name='Other'):
+    """
+    Combines all entries (name, count) with a count below the threshold and appends a new entry
+    """
+    a, b = [], []
+    for s in stat:
+        (a, b)[s[1] < threshold].append(s)
+    c = 0
+    for s in b:
+        c += s[1]
+    a.append([name, c])
+    return a
+
+
+def export_pie_chart(stat, file):
+    """
+
+    :param stat:
+    :return:
+    """
+    labels = [x[0] for x in stat]
+    sizes = [x[1] for x in stat]
+
+    fig, ax = plt.subplots(figsize=[4,4], tight_layout=True)
+    ax.pie(sizes, labels=labels, autopct='%1.1f%%', pctdistance=0.8, shadow=True, labeldistance=1.2)
+    # create output directory if necessary
+    containing_dir = os.path.dirname(file)
+    if not os.path.isdir(containing_dir):
+        os.mkdir(containing_dir)
+    plt.savefig(file, transparent=True)