From 32907d049867da693e099fdfc26bf23c94fa4b5c Mon Sep 17 00:00:00 2001 From: Trilarion Date: Wed, 6 Oct 2021 13:42:21 +0200 Subject: [PATCH] change ValueWithComment from composition to Inheritance (simplifies code a lot) --- code/archives.json | 1 + code/backlog.txt | 6 +- code/github_import.py | 10 +- code/gitlab_import.py | 8 +- code/html/generate_static_website.py | 230 ++++++++++++------ code/html/statistics.jinja | 5 +- code/maintenance_developers.py | 3 +- code/maintenance_entries.py | 72 +----- code/maintenance_inspirations.py | 1 - code/rejected.txt | 4 +- .../osgameclones_synchronization.py | 18 +- code/synchronization/sourceforge_import.py | 4 +- code/utils/constants.py | 5 +- code/utils/osg.py | 53 ++-- code/utils/osg_parse.py | 68 ++---- code/utils/osg_statistics.py | 21 +- entries/freee.md | 6 +- 17 files changed, 277 insertions(+), 238 deletions(-) diff --git a/code/archives.json b/code/archives.json index df06f7cd..aef63c14 100644 --- a/code/archives.json +++ b/code/archives.json @@ -702,6 +702,7 @@ "https://github.com/egordorichev/LastTry.git", "https://github.com/eguneys/lose-your-marbles.git", "https://github.com/eguneys/supaxl.git", + "https://github.com/ekolis/FrEee.git", "https://github.com/electronicarts/CnC_Remastered_Collection.git", "https://github.com/ellisonleao/clumsy-bird.git", "https://github.com/elnormous/ouzel.git", diff --git a/code/backlog.txt b/code/backlog.txt index 7fafcf0f..1cef926c 100644 --- a/code/backlog.txt +++ b/code/backlog.txt @@ -44,9 +44,11 @@ https://github.com/gecko0307/dagon https://github.com/googleforgames/open-match https://github.com/hiitiger/goverlay https://github.com/jmorton06/Lumos +https://github.com/loudsmilestudios/TetraForce https://github.com/open-telemetry/opentelemetry-cpp https://github.com/QodotPlugin/qodot-plugin https://github.com/Razakhel/RaZ +https://github.com/Relintai/broken_seals https://github.com/samdauwe/BabylonCpp https://github.com/SasLuca/rayfork https://github.com/Zylann/godot_heightmap_plugin @@ -96,6 +98,4 @@ https://www.seul.org/~grumbel/tmp/clanlib/games.html https://www.tapatalk.com/groups/imperilist/ https://www.wurfelengine.net/ https://zdoom.org/downloads (gzdoom, lzdoom) -https://zope.readthedocs.io/en/latest/ -https://github.com/loudsmilestudios/TetraForce -https://github.com/Relintai/broken_seals \ No newline at end of file +https://zope.readthedocs.io/en/latest/ \ No newline at end of file diff --git a/code/github_import.py b/code/github_import.py index 1f8056ec..5a237b14 100644 --- a/code/github_import.py +++ b/code/github_import.py @@ -82,7 +82,7 @@ def github_import(): # read entry entry = osg.read_entry(file) code_repositories = entry['Code repository'] - repos = [x.value for x in code_repositories if x.startswith(prefix)] + repos = [x for x in code_repositories if x.startswith(prefix)] repos[0] += ' @add' repos = [x for x in repos if '@add' in x] repos = [x.split(' ')[0] for x in repos] @@ -112,7 +112,7 @@ def github_import(): # update comment for r in code_repositories: - if r.value.startswith(repo): + if r.startswith(repo): break comments = r.comment if comments: @@ -128,7 +128,7 @@ def github_import(): language = info['language'] language = language_aliases.get(language, language) if language and language not in entry['Code language'] and language not in ignored_languages: - entry['Code language'].append(osg_parse.ValueWithComment(language)) + entry['Code language'].append(language) print(' added to languages: {}'.format(language)) # contributors @@ -155,7 +155,7 @@ def github_import(): # look up author in entry developers if name not in entry.get('Developer', []): print(' dev "{}" added to entry {}'.format(name, file)) - entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(name)] + entry['Developer'] = entry.get('Developer', []) + [name] # look up author in developers data base if name in all_developers: @@ -204,7 +204,7 @@ def github_starring_synchronization(): # get repos code_repositories = entry.get('Code repository', []) - repos = [x.value for x in code_repositories if x.startswith(prefix)] + repos = [x for x in code_repositories if x.startswith(prefix)] repos[0] += ' @add' repos = [x for x in repos if '@add' in x] repos = [x.split(' ')[0] for x in repos] diff --git a/code/gitlab_import.py b/code/gitlab_import.py index b22e1d8e..32bc476e 100644 --- a/code/gitlab_import.py +++ b/code/gitlab_import.py @@ -54,7 +54,7 @@ def gitlab_import(): # read entry entry = osg.read_entry(file) code_repositories = entry['Code repository'] - repos = [x.value for x in code_repositories if x.startswith(prefix)] + repos = [x for x in code_repositories if x.startswith(prefix)] repos[0] += ' @add' repos = [x for x in repos if '@add' in x] repos = [x.split(' ')[0] for x in repos] @@ -77,7 +77,7 @@ def gitlab_import(): # search for repository for r in code_repositories: - if r.value.startswith(repo): + if r.startswith(repo): break # update comment @@ -94,7 +94,7 @@ def gitlab_import(): # language in languages for language, usage in info['languages'].items(): if language in c.known_languages and usage > 5 and language not in entry['Code language']: - entry['Code language'].append(osg_parse.ValueWithComment(language)) + entry['Code language'].append(language) print(' added to languages: {}'.format(language)) entry['Code repository'] = code_repositories @@ -125,7 +125,7 @@ def gitlab_starring_synchronization(): # get repos code_repositories = entry.get('Code repository', []) - repos = [x.value for x in code_repositories if x.startswith(prefix)] + repos = [x for x in code_repositories if x.startswith(prefix)] repos[0] += ' @add' repos = [x for x in repos if '@add' in x] repos = [x.split(' ')[0] for x in repos] diff --git a/code/html/generate_static_website.py b/code/html/generate_static_website.py index 87274fba..a86eb8fd 100644 --- a/code/html/generate_static_website.py +++ b/code/html/generate_static_website.py @@ -80,7 +80,7 @@ import math import datetime import time from functools import partial -from utils import osg, constants as c, utils +from utils import osg, constants as c, utils, osg_statistics as stat, osg_parse from jinja2 import Environment, FileSystemLoader import html5lib @@ -97,13 +97,13 @@ alphabet_replacements = {'Á': 'A', 'Å': 'A', 'Ä': 'A', 'É': 'E', 'ⴹ': 'E', # the subfolder structure games_path = ['games'] -frameworks_path = ['frameworks'] +non_games_path = ['frameworks'] inspirations_path = ['inspirations'] developers_path = ['developers'] # derived paths games_index_path = games_path + ['index.html'] -frameworks_index_path = frameworks_path + ['index.html'] +non_games_index_path = non_games_path + ['index.html'] inspirations_index_path = inspirations_path + ['index.html'] developers_index_path = developers_path + ['index.html'] @@ -155,11 +155,12 @@ genre_icon_map = { # cross-references to the langue pages code_language_references = {l: games_by_language_path[:-1] + ['{}#{}'.format(games_by_language_path[-1], osg.canonical_name(l))] for l in c.known_languages} -# map of internal framework names to display names (which are in plural) -framework_names = { +# map of internal non game names to display names (which are in plural) +non_game_category_names = { 'tool': 'Tools', 'framework': 'Frameworks', - 'library': 'Libraries' + 'library': 'Libraries', + 'game engine': 'Game Engine' } # we check the output html structure every time @@ -284,15 +285,18 @@ def url_to(current, target, info=None): return url -def preprocess(list, key, url): +def preprocess(items, key, url): """ + Sets a few additional fields in the entries, inspirations, developers in order to generate the right content from + them later. - :param list: + :param url: + :param items: :param key: :return: """ - _ = set() - for item in list: + _ = set() # this is just to avoid duplicating anchors + for item in items: # add unique anchor ref anchor = osg.canonical_name(item[key]) while anchor in _: @@ -311,6 +315,11 @@ def preprocess(list, key, url): def game_index(entry): + """ + + :param entry: + :return: + """ e = { 'url': make_url(entry['href'], entry['Title']), 'anchor-id': entry['anchor-id'] @@ -326,6 +335,11 @@ def game_index(entry): def inspiration_index(inspiration): + """ + + :param inspiration: + :return: + """ e = { 'url': make_url(inspiration['href'], inspiration['Name']), 'anchor-id': inspiration['anchor-id'], @@ -337,6 +351,11 @@ def inspiration_index(inspiration): def developer_index(developer): + """ + + :param developer: + :return: + """ e = { 'url': make_url(developer['href'], developer['Name']), 'anchor-id': developer['anchor-id'] @@ -348,7 +367,12 @@ def developer_index(developer): def shortcut_url(url, name): + """ + :param url: + :param name: + :return: + """ # remove slash at the end if url.endswith('/'): url = url[:-1] @@ -388,6 +412,14 @@ def shortcut_url(url, name): def make_url(href, content, title=None, css_class=None): + """ + + :param href: + :param content: + :param title: + :param css_class: + :return: + """ if isinstance(content, str): content = make_text(content) url = { @@ -403,9 +435,15 @@ def make_url(href, content, title=None, css_class=None): def make_repo_url(x, name): + """ + + :param x: + :param name: + :return: + """ # parse comments comments = [] - if x.has_comment(): + if isinstance(x, osg_parse.Value): for c in x.comment.split(','): c = c.strip() if not c.startswith('@'): @@ -427,7 +465,7 @@ def make_repo_url(x, name): else: comments.append(make_icon('star-o', 'low rated')) # this is the default element - url = make_url(x.value, shortcut_url(x.value, name)) + url = make_url(x, shortcut_url(x, name)) if comments: return make_enumeration([url, make_enclose(make_enumeration(comments), '(', ')')], '') else: @@ -435,6 +473,12 @@ def make_repo_url(x, name): def make_icon(css_class, title=None): + """ + + :param css_class: + :param title: + :return: + """ icon = { 'type': 'icon', 'class': css_class, @@ -445,6 +489,12 @@ def make_icon(css_class, title=None): def make_text(content, css_class=None): + """ + + :param content: + :param css_class: + :return: + """ text = { 'type': 'text', 'text': content @@ -454,13 +504,14 @@ def make_text(content, css_class=None): return text -def make_nothing(): - return { - 'type': 'nothing' - } - - def make_enclose(entry, left, right): + """ + + :param entry: + :param left: + :param right: + :return: + """ enclose = { 'type': 'enclose', 'entry': entry, @@ -471,6 +522,12 @@ def make_enclose(entry, left, right): def make_enumeration(entries, divider=', '): + """ + + :param entries: + :param divider: + :return: + """ enumeration = { 'type': 'enumeration', 'entries': entries, @@ -480,6 +537,11 @@ def make_enumeration(entries, divider=', '): def make_tags(entries): + """ + + :param entries: + :return: + """ return { 'type': 'tags', 'enumeration': make_enumeration(entries, divider='') @@ -505,6 +567,12 @@ def developer_profile_link(link): def convert_inspirations(inspirations, entries): + """ + + :param inspirations: + :param entries: + :return: + """ entries_references = {entry['Title']: entry['href'] for entry in entries} for inspiration in inspirations: name = inspiration['Name'] @@ -524,6 +592,12 @@ def convert_inspirations(inspirations, entries): def convert_developers(developers, entries): + """ + + :param developers: + :param entries: + :return: + """ entries_references = {entry['Title']:entry['href'] for entry in entries} for developer in developers: name = developer['Name'] @@ -552,9 +626,14 @@ def convert_developers(developers, entries): def create_keyword_tag(keyword): + """ + + :param keyword: + :return: + """ if keyword in c.recommended_keywords: if keyword in c.non_game_keywords: - url = frameworks_index_path.copy() + url = non_games_index_path.copy() else: url = games_by_genres_path.copy() url[-1] += '#{}'.format(keyword) @@ -567,6 +646,11 @@ def create_keyword_tag(keyword): def create_state_texts(states): + """ + + :param states: + :return: + """ texts = [] if 'mature' in states: texts.append(make_text('mature', 'has-text-weight-bold')) @@ -581,6 +665,13 @@ def create_state_texts(states): def convert_entries(entries, inspirations, developers): + """ + + :param entries: + :param inspirations: + :param developers: + :return: + """ inspirations_references = {inspiration['Name']: inspiration['href'] for inspiration in inspirations} developer_references = {developer['Name']: developer['href'] for developer in developers} for entry in entries: @@ -596,7 +687,7 @@ def convert_entries(entries, inspirations, developers): entry['note'] = make_text(entry['Note'], 'is-italic') # keywords as tags - e = [create_keyword_tag(x.value) for x in entry['Keyword']] + e = [create_keyword_tag(x) for x in entry['Keyword']] entry['keyword'] = make_tags(e) # other normal fields (not technical info) @@ -604,8 +695,6 @@ def convert_entries(entries, inspirations, developers): if field in entry: e = entry[field] divider = ', ' - if isinstance(e[0], osg.osg_parse.ValueWithComment): - e = [x.value for x in e] if field == 'Inspiration': e = [make_url(inspirations_references[x], make_text(x)) for x in e] elif field == 'Developer': @@ -627,8 +716,6 @@ def convert_entries(entries, inspirations, developers): # platforms if 'Platform' in entry: e = entry['Platform'] - if isinstance(e[0], osg.osg_parse.ValueWithComment): - e = [x.value for x in e] else: e = ['Unspecified'] e = [make_url(games_by_platform_path[:-1] + ['{}#{}'.format(games_by_platform_path[-1], x.lower())], make_icon(platform_icon_map[x]), x) if x in platform_icon_map else make_text(x) for x in e] @@ -643,8 +730,6 @@ def convert_entries(entries, inspirations, developers): divider = ', ' if not e: continue - if isinstance(e[0], osg.osg_parse.ValueWithComment) and field != 'Code repository': - e = [x.value for x in e] if field == 'Code repository': e = [make_repo_url(x, name) for x in e] elif field == 'Code language': @@ -663,8 +748,6 @@ def convert_entries(entries, inspirations, developers): if field in entry['Building']: e = entry['Building'][field] divider = ', ' - if isinstance(e[0], osg.osg_parse.ValueWithComment): - e = [x.value for x in e] e = [make_url(c.build_system_urls[x], x, css_class='is-size-7') if x in c.build_system_urls else make_text(x, 'is-size-7') for x in e] namex = make_text('{}: '.format(field), 'is-size-7') entry[field.lower()] = [namex, make_enumeration(e, divider)] @@ -673,19 +756,29 @@ def convert_entries(entries, inspirations, developers): def add_license_links_to_entries(entries): + """ + + :param entries: + :return: + """ for entry in entries: licenses = entry['Code license'] - licenses = [(c.license_urls.get(license.value, ''), license.value) for license in licenses] + licenses = [(c.license_urls.get(license, ''), license) for license in licenses] entry['Code license'] = licenses def get_top50_games(games): + """ + + :param games: + :return: + """ top50_games = [] for game in games: # get stars of repositories stars = 0 for repo in game.get('Code repository', []): - if repo.has_comment(): + if isinstance(repo, osg_parse.Value): for c in repo.comment.split(','): c = c.strip() if not c.startswith('@'): @@ -707,25 +800,24 @@ def get_top50_games(games): def generate(entries, inspirations, developers): """ - - :param entries: - :param inspirations: - :param developers: + Regenerates the whole static website given an already imported set of entries, inspirations and developers. + These datasets must be valid for each other, i.e. each inspiration listed in entries must also have an + entry in inspirations and the same holds for developers. """ - # split entries in games and frameworks - games, frameworks = [], [] + # split entries in games and non-games + games, non_games = [], [] for entry in entries: - (games, frameworks)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry) + (games, non_games)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry) # preprocess preprocess(games, 'Title', games_path) - preprocess(frameworks, 'Title', frameworks_path) + preprocess(non_games, 'Title', non_games_path) # TODO preprocess doesn't set the urls for frameworks correctly fix here, do better later - for framework in frameworks: - keyword = [keyword for keyword in c.non_game_keywords if keyword in framework['Keyword']][0] - framework['href'] = frameworks_path + ['{}.html#{}'.format(keyword, framework['anchor-id'])] - entries = games + frameworks + for non_game in non_games: + keyword = [keyword for keyword in c.non_game_keywords if keyword in non_game['Keyword']][0] + non_game['href'] = non_games_path + ['{}.html#{}'.format(keyword, non_game['anchor-id'])] + entries = games + non_games preprocess(inspirations, 'Name', inspirations_path) preprocess(developers, 'Name', developers_path) @@ -733,7 +825,7 @@ def generate(entries, inspirations, developers): convert_inspirations(inspirations, entries) convert_developers(developers, entries) convert_entries(games, inspirations, developers) - convert_entries(frameworks, inspirations, developers) + convert_entries(non_games, inspirations, developers) # set external links up add_license_links_to_entries(games) @@ -749,7 +841,7 @@ def generate(entries, inspirations, developers): games_by_genre = sort_into_categories(games, genres, lambda item, category: category.lower() in item['Keyword']) games_by_platform = sort_into_categories(entries, c.valid_platforms, lambda item, category: category in item.get('Platform', []), 'Unspecified') games_by_language = sort_into_categories(entries, c.known_languages, lambda item, category: category in item['Code language']) - frameworks_by_type = sort_into_categories(frameworks, c.non_game_keywords, lambda item, category: category in item['Keyword']) + non_games_by_type = sort_into_categories(non_games, c.non_game_keywords, lambda item, category: category in item['Keyword']) # extract top 50 Github stars games top50_games = get_top50_games(games) @@ -783,19 +875,19 @@ def generate(entries, inspirations, developers): # index.html base['active_nav'] = 'index' - index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(frameworks))) } + index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(non_games))) } template = environment.get_template('index.jinja') write(template.render(index=index), ['index.html']) - # contribute.html + # contribute page base['active_nav'] = 'contribute' template = environment.get_template('contribute.jinja') write(template.render(), ['contribute.html']) - # statistics + # statistics page base['active_nav'] = 'statistics' - # preparation + # statistics preparation template = environment.get_template('statistics.jinja') data = { 'title': 'Statistics', @@ -803,48 +895,42 @@ def generate(entries, inspirations, developers): } # build-systems - build_systems = [] - field = 'Build system' - for entry in entries: - if field in entry['Building']: - build_systems.extend(entry['Building'][field]) - build_systems = [x.value for x in build_systems] - - unique_build_systems = set(build_systems) - unique_build_systems = [(l, build_systems.count(l)) for l in unique_build_systems] - unique_build_systems.sort(key=lambda x: str.casefold(x[0])) # first sort by name - unique_build_systems.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first) + build_systems_stat = stat.get_build_systems(entries) + # TODO replace all entries < 10 with "others" + # TODO make Pie chart out of it section = { 'title': 'Build system', - 'items': ['{} ({})'.format(*item) for item in unique_build_systems] + 'items': ['{} ({})'.format(*item) for item in build_systems_stat] } data['sections'].append(section) + + # render and write statistics page write(template.render(data=data), ['statistics.html']) - # frameworks folder - base['url_to'] = partial(url_to, frameworks_path) + # non-games folder + base['url_to'] = partial(url_to, non_games_path) base['active_nav'] = 'frameworks' - # frameworks by type - index = divide_in_three_columns_and_transform(frameworks_by_type, game_index) + # non-games by type + index = divide_in_three_columns_and_transform(non_games_by_type, game_index) index['title'] = make_text('Open source game frameworks/tools') - index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(frameworks))) + index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(non_games))) index['categories'] = c.non_game_keywords - index['category-names'] = framework_names + index['category-names'] = non_game_category_names index['category-icons'] = {} index['number_entries_per_category_threshold'] = 0 index['entry_bold'] = lambda x: 'tags' not in x index['category-infos'] = {} - write(template_categorical_index.render(index=index), frameworks_index_path) + write(template_categorical_index.render(index=index), non_games_index_path) - # generate frameworks pages + # generate non-games pages for keyword in c.non_game_keywords: listing = { - 'title': framework_names[keyword], - 'subtitle': make_url(frameworks_index_path, 'Index'), - 'items': frameworks_by_type[keyword] + 'title': non_game_category_names[keyword], + 'subtitle': make_url(non_games_index_path, 'Index'), + 'items': non_games_by_type[keyword] } - write(template_listing_entries.render(listing=listing), frameworks_path +['{}.html'.format(keyword)]) + write(template_listing_entries.render(listing=listing), non_games_path + ['{}.html'.format(keyword)]) # games folder base['url_to'] = partial(url_to, games_path) diff --git a/code/html/statistics.jinja b/code/html/statistics.jinja index de44eff5..9bbe51a7 100644 --- a/code/html/statistics.jinja +++ b/code/html/statistics.jinja @@ -1,12 +1,13 @@ {% extends "base.jinja" %} {% block content %} -
-

{{ data['title'] }}

+
+

{{ data['title'] }}

{% set comma = joiner(",") %} {% for section in data['sections'] %} {{ comma() }} {{ section['title'] }} {% endfor %}
+ {#- each statistics section -#} {% for section in data['sections'] %}
diff --git a/code/maintenance_developers.py b/code/maintenance_developers.py index 0114fc9c..f1675045 100644 --- a/code/maintenance_developers.py +++ b/code/maintenance_developers.py @@ -100,7 +100,6 @@ class DevelopersMaintainer: entry_name = entry['Title'] entry_devs = entry.get('Developer', []) for entry_dev in entry_devs: - entry_dev = entry_dev.value # ignore a possible comment if entry_dev in self.developers: self.developers[entry_dev]['Games'].append(entry_name) else: @@ -127,7 +126,7 @@ class DevelopersMaintainer: # for entry in self.entries: # for developer in entry.get('Developer', []): # if developer.comment: - # print('{:<25} - {:<25} - {}'.format(entry['File'], developer.value, developer.comment)) + # print('{:<25} - {:<25} - {}'.format(entry['File'], developer, developer.comment)) if __name__ == "__main__": diff --git a/code/maintenance_entries.py b/code/maintenance_entries.py index 8f649e86..a85d7d39 100644 --- a/code/maintenance_entries.py +++ b/code/maintenance_entries.py @@ -56,7 +56,6 @@ def create_toc(title, file, entries): rows = [] for entry in entries: info = entry['Code language'] + entry['Code license'] + entry['State'] - info = [x.value for x in info] rows.append('- **[{}]({})** ({})'.format(entry['Title'], '../' + entry['File'], ', '.join(info))) # sort rows (by title) @@ -130,7 +129,6 @@ class EntriesMaintainer: keywords.extend(entry['Keyword']) if b'first\xe2\x80\x90person'.decode() in entry['Keyword']: print(entry['File']) - keywords = [x.value for x in keywords] # reduce those starting with "multiplayer" keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords] @@ -159,7 +157,6 @@ class EntriesMaintainer: for entry in self.entries: deps = entry.get('Code dependency', []) for dependency in deps: - dependency = dependency.value if dependency in referenced_dependencies: referenced_dependencies[dependency] += 1 else: @@ -516,7 +513,6 @@ class EntriesMaintainer: languages = [] for entry in self.entries: languages.extend(entry[field]) - languages = [x.value for x in languages] unique_languages = set(languages) unique_languages = [(l, languages.count(l) / len(languages)) for l in unique_languages] @@ -538,7 +534,6 @@ class EntriesMaintainer: licenses = [] for entry in self.entries: licenses.extend(entry[field]) - licenses = [x.value for x in licenses] unique_licenses = set(licenses) unique_licenses = [(l, licenses.count(l) / len(licenses)) for l in unique_licenses] @@ -560,7 +555,6 @@ class EntriesMaintainer: keywords = [] for entry in self.entries: keywords.extend(entry[field]) - keywords = [x.value for x in keywords] # reduce those starting with "multiplayer" keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords] @@ -597,7 +591,7 @@ class EntriesMaintainer: popular = False for repo in entry.get(field, []): for popular_repo in popular_code_repositories: - if popular_repo in repo.value: + if popular_repo in repo: popular = True break # if there were repositories, but none popular, add them to the list @@ -618,7 +612,6 @@ class EntriesMaintainer: if field in entry: code_dependencies.extend(entry[field]) entries_with_code_dependency += 1 - code_dependencies = [x.value for x in code_dependencies] statistics += 'With code dependency field {} ({:.1f}%)\n\n'.format(entries_with_code_dependency, rel(entries_with_code_dependency)) @@ -644,7 +637,6 @@ class EntriesMaintainer: for entry in self.entries: if field in entry['Building']: build_systems.extend(entry['Building'][field]) - build_systems = [x.value for x in build_systems] statistics += 'Build systems information available for {:.1f}% of all projects.\n\n'.format( rel(len(build_systems))) @@ -690,7 +682,6 @@ class EntriesMaintainer: for entry in self.entries: if field in entry: platforms.extend(entry[field]) - platforms = [x.value for x in platforms] statistics += 'Platform information available for {:.1f}% of all projects.\n\n'.format(rel(len(platforms))) @@ -708,67 +699,12 @@ class EntriesMaintainer: def update_html(self): """ - Parses all entries, collects interesting info and stores it in a json file suitable for displaying - with a dynamic table in a browser. """ if not self.entries: print('entries not yet loaded') return - # make database out of it - db = {'headings': ['Game', 'Description', 'Download', 'State', 'Keyword', 'Source']} - - entries = [] - for info in self.entries: - - # game & description - entry = ['{} (home, entry)'.format(info['Title'], info['Home'][0], - r'https://github.com/Trilarion/opensourcegames/blob/master/entries/' + - info['File']), - textwrap.shorten(info.get('Note', ''), width=60, placeholder='..')] - - # download - field = 'Download' - if field in info and info[field]: - entry.append('Link'.format(info[field][0])) - else: - entry.append('') - - # state (field state is essential) - entry.append('{} / {}'.format(info['State'][0], - 'inactive since {}'.format(osg.extract_inactive_year(info)) if osg.is_inactive(info) else 'active')) - - # keywords - keywords = info['Keyword'] - keywords = [x.value for x in keywords] - entry.append(', '.join(keywords)) - - # source - text = [] - field = 'Code repository' - if field in info and info[field]: - text.append('Source'.format(info[field][0].value)) - languages = info['Code language'] - languages = [x.value for x in languages] - text.append(', '.join(languages)) - licenses = info['Code license'] - licenses = [x.value for x in licenses] - text.append(', '.join(licenses)) - entry.append(' - '.join(text)) - - # append to entries - entries.append(entry) - - # sort entries by game name - entries.sort(key=lambda x: str.casefold(x[0])) - - db['data'] = entries - - # output - text = json.dumps(db, indent=1) - utils.write_text(c.json_db_file, text) - - print('HTML updated') + print('HTML not updated') def update_repos(self): """ @@ -784,7 +720,6 @@ class EntriesMaintainer: # for every entry filter those that are known git repositories (add additional repositories) for entry in self.entries: repos = entry.get('Code repository', []) - repos = [x.value for x in repos] # keep the first and all others containing @add if not repos: continue @@ -841,7 +776,6 @@ class EntriesMaintainer: git_repos = [] for entry in self.entries: repos = entry['Code repository'] - repos = [x.value for x in repos] for repo in repos: repo = repo.split(' ')[0].strip() url = osg.git_repo(repo) @@ -898,7 +832,7 @@ class EntriesMaintainer: # stats = {} # for entry in self.entries: # repos = entry.get('Code repository', []) - # comments = [x.comment for x in repos if x.value.startswith('https://github.com/') and x.comment] + # comments = [x.comment for x in repos if x.startswith('https://github.com/') and x.comment] # for comment in comments: # for part in comment.split(','): # part = part.strip() diff --git a/code/maintenance_inspirations.py b/code/maintenance_inspirations.py index 84330fb5..eb827b8f 100644 --- a/code/maintenance_inspirations.py +++ b/code/maintenance_inspirations.py @@ -152,7 +152,6 @@ class InspirationMaintainer: for entry in self.entries: entry_name = entry['Title'] for inspiration in entry.get('Inspiration', []): - inspiration = inspiration.value if inspiration in self.inspirations: self.inspirations[inspiration]['Inspired entries'].append(entry_name) else: diff --git a/code/rejected.txt b/code/rejected.txt index bbd43159..eb4f5f21 100644 --- a/code/rejected.txt +++ b/code/rejected.txt @@ -79,6 +79,7 @@ HistoryLine (https://github.com/oliverdb/Historyline): Very early development, n Homeworld SDL (https://github.com/aheadley/homeworld): Not open source compliant license (see https://github.com/aheadley/homeworld/blob/master/README) Howitzer Skirmish (http://howski.sourceforge.net/): No source code Ikariam (https://github.com/advocaite/ikariam, https://github.com/TheOnly92/Ikariem): No license information found, no assets license information found +Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files) imperialism-remake (http://remake.twelvepm.de/, https://github.com/Trilarion/imperialism-Remake): Too minimal (I must know) Imperium: Sticks (http://rtciv.sourceforge.net/, http://sourceforge.net/projects/rtciv): No source code available ImperiumAO (https://sourceforge.net/projects/impao, https://www.imperiumao.com.ar/): Only engine is open source, engine is ORE @@ -209,5 +210,4 @@ XQuest 2 (http://www.swallowtail.org/xquest/, http://www.swallowtail.org/xquest/ xrick (http://www.bigorno.net/xrick): No open source license/unclear license (see file README in http://www.bigorno.net/xrick/xrick-021212.zip) Yave (https://github.com/gan74/Yave): General graphics engine, not game centered in any way and experimental Yuris Revenge (https://github.com/cookgreen/Yuris-Revenge): Mod to OpenRA -zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance -Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files) \ No newline at end of file +zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance \ No newline at end of file diff --git a/code/synchronization/osgameclones_synchronization.py b/code/synchronization/osgameclones_synchronization.py index 114020c1..9e5b99c6 100644 --- a/code/synchronization/osgameclones_synchronization.py +++ b/code/synchronization/osgameclones_synchronization.py @@ -281,14 +281,14 @@ if __name__ == "__main__": osgc_languages = osgc_entry['lang'] if type(osgc_languages) == str: osgc_languages = [osgc_languages] - our_languages = [x.value for x in our_entry['Code language']] # essential field + our_languages = our_entry['Code language'] p += compare_sets(osgc_languages, our_languages, 'code language') # compare their license with our code and assets license if 'license' in osgc_entry: osgc_licenses = osgc_entry['license'] - our_code_licenses = [x.value for x in our_entry['Code license']] # essential field - our_assets_licenses = [x.value for x in our_entry.get('Assets license', [])] + our_code_licenses = our_entry['Code license'] # essential field + our_assets_licenses = our_entry.get('Assets license', []) p += compare_sets(osgc_licenses, our_code_licenses + our_assets_licenses, 'licenses', 'notthem') p += compare_sets(osgc_licenses, our_code_licenses, 'licenses', 'notus') @@ -298,7 +298,7 @@ if __name__ == "__main__": osgc_frameworks = osgc_entry['framework'] if type(osgc_frameworks) == str: osgc_frameworks = [osgc_frameworks] - our_frameworks = [x.value for x in our_entry.get('Code dependency', [])] + our_frameworks = our_entry.get('Code dependency', []) our_frameworks = [x.casefold() for x in our_frameworks] our_frameworks = [x if x not in our_framework_replacements else our_framework_replacements[x] for x in our_frameworks] @@ -315,13 +315,13 @@ if __name__ == "__main__": 'sourceforge.net/projects/')] # we don't need the general sites there # osgc_repos = [x for x in osgc_repos if not x.startswith('https://sourceforge.net/projects/')] # ignore some our_repos = our_entry.get('Code repository', []) - our_repos = [utils.strip_url(url.value) for url in our_repos] + our_repos = [utils.strip_url(url) for url in our_repos] our_repos = [x for x in our_repos if not x.startswith( 'gitlab.com/osgames/')] # we do not yet spread our own deeds (but we will some day) our_repos = [x for x in our_repos if 'cvs.sourceforge.net' not in x and 'svn.code.sf.net/p/' not in x] # no cvs or svn anymore our_downloads = our_entry.get('Download', []) - our_downloads = [utils.strip_url(url.value) for url in our_downloads] + our_downloads = [utils.strip_url(url) for url in our_downloads] p += compare_sets(osgc_repos, our_repos + our_downloads, 'repo', 'notthem') # if their repos are not in our downloads or repos p += compare_sets(osgc_repos, our_repos[:1], 'repo', @@ -334,7 +334,7 @@ if __name__ == "__main__": osgc_urls = [osgc_urls] osgc_urls = [utils.strip_url(url) for url in osgc_urls] our_urls = our_entry['Home'] - our_urls = [utils.strip_url(url.value) for url in our_urls] + our_urls = [utils.strip_url(url) for url in our_urls] p += compare_sets(osgc_urls, our_urls, 'url/home', 'notthem') # if their urls are not in our urls # our_urls = [url for url in our_urls if # not url.startswith('github.com/')] # they don't have them as url @@ -361,10 +361,10 @@ if __name__ == "__main__": p += ' development : mismatch : them complete, us not mature\n' # get our keywords - our_keywords = [x.value for x in our_entry['Keyword']] # essential + our_keywords = our_entry['Keyword'] # essential # compare their originals to our inspirations - our_originals = [x.value for x in our_entry.get('Inspiration', [])] + our_originals = our_entry.get('Inspiration', []) if 'originals' in osgc_entry: osgc_originals = osgc_entry['originals'] osgc_originals = [x.replace(',', '') for x in diff --git a/code/synchronization/sourceforge_import.py b/code/synchronization/sourceforge_import.py index 2e3fac46..b525e667 100644 --- a/code/synchronization/sourceforge_import.py +++ b/code/synchronization/sourceforge_import.py @@ -76,7 +76,7 @@ def sourceforge_import(): # read full entry entry = osg.read_entry(file) developers = entry.get('Developer', []) - urls = [x.value for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')] + urls = [x for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')] # do we need to save it again entry_changed = False @@ -133,7 +133,7 @@ def sourceforge_import(): # look author up in entry developers field, if not existing add if author_name not in developers: print(' dev "{}" added to entry {}'.format(author_name, file)) - entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(author_name)] + entry['Developer'] = entry.get('Developer', []) + [author_name] entry_changed = True developers = entry.get('Developer', []) # update developers diff --git a/code/utils/constants.py b/code/utils/constants.py index a2c5ee14..7d661a7a 100644 --- a/code/utils/constants.py +++ b/code/utils/constants.py @@ -68,9 +68,10 @@ fields_without_comments = ('Inspiration', 'Play', 'Download', 'Platform', 'Code recommended_keywords = ( 'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation', 'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake') +# TODO unmake remake a recommended keyword (should be the same as clone maybe), i.e. add another recommended keyword if only remake is in there -# non game keywords take precedence -non_game_keywords = ('framework', 'library', 'tool') +# non game keywords take precedence over other (game) recommended keywords, at most one of them per entry +non_game_keywords = ('framework', 'game engine', 'library', 'tool') # known programming languages, anything else will result in a warning during a maintenance operation # only these will be used when gathering statistics diff --git a/code/utils/osg.py b/code/utils/osg.py index a03b3c8a..913df80c 100644 --- a/code/utils/osg.py +++ b/code/utils/osg.py @@ -340,11 +340,11 @@ def check_and_process_entry(entry): if canonical_file_name != file and canonical_file_name != file[:-5] + '.md': message += 'file name should be {}\n'.format(canonical_file_name) - # check that fields without comments have no comments, set to field without comment + # check that fields without comments have no comments (i.e. are no Values) for field in c.fields_without_comments: if field in entry: content = entry[field] - if any(item.has_comment() for item in content): + if any(isinstance(item, osg_parse.Value) for item in content): message += 'field without comments {} has comment\n'.format(field) # state must contain either beta or mature but not both @@ -359,8 +359,8 @@ def check_and_process_entry(entry): for field in c.url_fields: values = entry.get(field, []) for value in values: - if value.value.startswith('<') and value.value.endswith('>'): - value.value = value.value[1:-1] + if value.startswith('<') and value.endswith('>'): + value = value[1:-1] if not any(value.startswith(x) for x in c.valid_url_prefixes): message += 'URL "{}" in field "{}" does not start with a valid prefix'.format(value, field) @@ -368,7 +368,7 @@ def check_and_process_entry(entry): for repo in entry.get('Code repository', []): if any(repo.startswith(x) for x in ('@', '?')): continue - repo = repo.value.split(' ')[0].strip() + repo = repo.split(' ')[0].strip() if any((x in repo for x in ('github', 'gitlab', 'git.tuxfamily', 'git.savannah'))): if not repo.startswith('https://'): message += 'Repo "{}" should start with https://'.format(repo) @@ -420,7 +420,7 @@ def is_inactive(entry): def extract_inactive_year(entry): state = entry['State'] phrase = 'inactive since ' - inactive_year = [x.value[len(phrase):] for x in state if x.startswith(phrase)] + inactive_year = [x[len(phrase):] for x in state if x.startswith(phrase)] assert len(inactive_year) <= 1 if inactive_year: return int(inactive_year[0]) @@ -457,9 +457,29 @@ def write_entry(entry): utils.write_text(entry_path, content) -def create_entry_content(entry): +def render_value(value): """ + :param value: + :return: + """ + if isinstance(value, osg_parse.Value): + comment = value.comment + else: + comment = None + if any(x in value for x in (',', ' (')): + value = '"{}"'.format(value) + if comment: + return '{} ({})'.format(value, comment) + else: + return value + + +def create_entry_content(entry): + """ + Creates the entry content from an internal representation as dictionary with fields to a text file representation + that can be stored in the md files. It should be compatible with the gramar and reading a file and re-creating the + content should not change the content. Importanly, the comments of the values have to be added here. :param entry: :return: """ @@ -468,7 +488,7 @@ def create_entry_content(entry): content = '# {}\n\n'.format(entry['Title']) # we automatically sort some fields - sort_fun = lambda x: str.casefold(x.value) + sort_fun = lambda x: str.casefold(x) for field in ('Media', 'Inspiration', 'Code Language', 'Developer', 'Build system'): if field in entry: values = entry[field] @@ -479,12 +499,11 @@ def create_entry_content(entry): b = [x for x in keywords if x not in c.recommended_keywords] entry['Keyword'] = sorted(a, key=sort_fun) + sorted(b, key=sort_fun) - # now all properties in the recommended order + # now all properties are in the recommended order for field in c.valid_properties: if field in entry: e = entry[field] - e = ['"{}"'.format(x) if any(y in x.value for y in (',', ' (')) else x for x in e] - e = [str(x) for x in e] + e = [render_value(x) for x in e] e = list(dict.fromkeys(e)) # this removes duplicates while keeping the sorting order content += '- {}: {}\n'.format(field, ', '.join(e)) content += '\n' @@ -504,8 +523,8 @@ def create_entry_content(entry): has_properties = True content += '\n' e = entry['Building'][field] - e = ['"{}"'.format(x) if ',' in x else x for x in e] - e = [str(x) for x in e] + e = [render_value(x) for x in e] + e = list(dict.fromkeys(e)) # this removes duplicates while keeping the sorting order content += '- {}: {}\n'.format(field, ', '.join(e)) # if there is a note, insert it @@ -533,16 +552,14 @@ def all_urls(entries): :param entries: :return: """ + # TODO there are other fields than c.url_fields and also in comments, maybe just regex on the whole content + # TODO this might be part of the external link check or it might not, check for duplicate code urls = {} # iterate over entries for entry in entries: file = entry['File'] - for field in c.url_fields: # TODO there are other fields, maybe just regex on the whole content + for field in c.url_fields: for value in entry.get(field, []): - if value.comment: - value = value.value + ' ' + value.comment - else: - value = value.value for subvalue in value.split(' '): subvalue = subvalue.strip() if is_url(subvalue): diff --git a/code/utils/osg_parse.py b/code/utils/osg_parse.py index 444d14fb..ae4cada2 100644 --- a/code/utils/osg_parse.py +++ b/code/utils/osg_parse.py @@ -14,10 +14,10 @@ class ListingTransformer(lark.Transformer): """ def unquoted_value(self, x): - return x[0].value.strip() + return x[0].strip() def quoted_value(self, x): - return x[0].value[1:-1].strip() # remove quotation marks and strip whitespaces + return x[0][1:-1].strip() # remove quotation marks and strip whitespaces def property(self, x): """ @@ -25,7 +25,7 @@ class ListingTransformer(lark.Transformer): :param x: :return: """ - return x[0].value, x[1:] + return x[0], x[1:] def name(self, x): """ @@ -33,7 +33,7 @@ class ListingTransformer(lark.Transformer): :param x: :return: """ - return 'Name', x[0].value.strip() + return 'Name', x[0].strip() def entry(self, x): """ @@ -56,19 +56,25 @@ class ListingTransformer(lark.Transformer): class EntryTransformer(lark.Transformer): def unquoted_value(self, x): - return x[0].value.strip() + return x[0].strip() def quoted_value(self, x): - return x[0].value[1:-1].strip() # remove quotation marks + return x[0][1:-1].strip() # remove quotation marks def comment_value(self, x): - return x[0].value[1:-1].strip() # remove parenthesis + return x[0][1:-1].strip() # remove parenthesis def value(self, x): + """ + This also stores the comment if needed. + + :param x: + :return: + """ if len(x) == 1: - v = ValueWithComment(value=x[0]) + v = x[0] else: - v = ValueWithComment(value=x[0], comment=x[1]) + v = Value(*x) return v def property(self, x): @@ -77,10 +83,10 @@ class EntryTransformer(lark.Transformer): :param x: :return: """ - return x[0].value.strip(), x[1:] + return x[0].strip(), x[1:] def title(self, x): - return 'Title', x[0].value.strip() + return 'Title', x[0].strip() def note(self, x): """ @@ -90,7 +96,7 @@ class EntryTransformer(lark.Transformer): """ if not x: raise lark.Discard - return 'Note', ''.join((x.value for x in x)) + return 'Note', ''.join(x) def building(self, x): return 'Building', x @@ -98,40 +104,16 @@ class EntryTransformer(lark.Transformer): def start(self, x): return x -# TODO turns out ValueWithComment does not really solve problem but actually creates even some, are there alternatives like inheriting from string? -class ValueWithComment: + +class Value(str): """ - All our property values can have (optional) comments. This is the class that represents them to us and implements - equality and 'in' operator functionality purely on the value. + A value is a string with some additional meta object (a comment) but mostly behaves as a string. """ - def __init__(self, value, comment=None): - self.value = value - self.comment = comment - - def is_empty(self): - return self.value == '' - - def has_comment(self): - return self.comment is not None - - def startswith(self, str): - return self.value.startswith(str) - - def __contains__(self, item): - return item in self.value - - def __eq__(self, other): - return self.value == other - - def __repr__(self): - if self.comment: - return '{} ({})'.format(self.value, self.comment) - else: - return '{}'.format(self.value) - - def __hash__(self): - return hash(self.value) + def __new__(cls, value, comment): + obj = str.__new__(cls, value) + obj.comment = comment + return obj def parse(parser, transformer, content): tree = parser.parse(content) diff --git a/code/utils/osg_statistics.py b/code/utils/osg_statistics.py index d579200d..b4482aa7 100644 --- a/code/utils/osg_statistics.py +++ b/code/utils/osg_statistics.py @@ -1,4 +1,23 @@ """ +Central place to calculate statistics about the entries. Used for updating the statistics.md file and the statistics page +of the website. +""" -""" +def get_build_systems(entries): + """ + Given a list of entries, calculates statistics about the used build systems and returns the statistics as + sorted list of elements (build-system-name, occurence). + "n/a" is used if no build system was specified + """ + build_systems = [] + for entry in entries: + build_systems.extend(entry['Building'].get('Build system', ['n/a'])) + + unique_build_systems = set(build_systems) + + build_systems_stat = [(l, build_systems.count(l)) for l in unique_build_systems] + build_systems_stat.sort(key=lambda x: str.casefold(x[0])) # first sort by name + build_systems_stat.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first) + + return build_systems_stat diff --git a/entries/freee.md b/entries/freee.md index 61e2d416..1e50a683 100644 --- a/entries/freee.md +++ b/entries/freee.md @@ -2,9 +2,9 @@ - Home: http://edkolis.com/freee - Inspiration: Space Empires IV -- State: beta +- State: beta - Download: https://github.com/ekolis/FrEee/releases -- Keyword: strategy, turn-based, space, content open +- Keyword: strategy, content open, space, turn-based - Code repository: https://github.com/ekolis/FrEee.git - Code language: C# - Code license: CC-BY-NC-SA-2.0 (https://github.com/ekolis/FrEee/blob/master/FrEee.Assets/Licenses/license.txt) @@ -12,4 +12,4 @@ ## Building -- Build system: Visual Studio \ No newline at end of file +- Build system: Visual Studio