AnotherFoxGuy/opensourcegames
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

change ValueWithComment from composition to Inheritance (simplifies code a lot)

Browse Source
This commit is contained in:
Trilarion 2021-10-06 13:42:21 +02:00
parent b291102272
commit 32907d0498
17 changed files with 277 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
code/archives.json
View File

@ -702,6 +702,7 @@
"https://github.com/egordorichev/LastTry.git",
"https://github.com/eguneys/lose-your-marbles.git",
"https://github.com/eguneys/supaxl.git",
"https://github.com/ekolis/FrEee.git",
"https://github.com/electronicarts/CnC_Remastered_Collection.git",
"https://github.com/ellisonleao/clumsy-bird.git",
"https://github.com/elnormous/ouzel.git",

6
code/backlog.txt
View File

@ -44,9 +44,11 @@ https://github.com/gecko0307/dagon
https://github.com/googleforgames/open-match
https://github.com/hiitiger/goverlay
https://github.com/jmorton06/Lumos
https://github.com/loudsmilestudios/TetraForce
https://github.com/open-telemetry/opentelemetry-cpp
https://github.com/QodotPlugin/qodot-plugin
https://github.com/Razakhel/RaZ
https://github.com/Relintai/broken_seals
https://github.com/samdauwe/BabylonCpp
https://github.com/SasLuca/rayfork
https://github.com/Zylann/godot_heightmap_plugin
@ -96,6 +98,4 @@ https://www.seul.org/~grumbel/tmp/clanlib/games.html
https://www.tapatalk.com/groups/imperilist/
https://www.wurfelengine.net/
https://zdoom.org/downloads (gzdoom, lzdoom)
https://zope.readthedocs.io/en/latest/
https://github.com/loudsmilestudios/TetraForce
https://github.com/Relintai/broken_seals
https://zope.readthedocs.io/en/latest/

10
code/github_import.py
View File

@ -82,7 +82,7 @@ def github_import():
# read entry
entry = osg.read_entry(file)
code_repositories = entry['Code repository']
repos = [x.value for x in code_repositories if x.startswith(prefix)]
repos = [x for x in code_repositories if x.startswith(prefix)]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]
@ -112,7 +112,7 @@ def github_import():
# update comment
for r in code_repositories:
if r.value.startswith(repo):
if r.startswith(repo):
break
comments = r.comment
if comments:
@ -128,7 +128,7 @@ def github_import():
language = info['language']
language = language_aliases.get(language, language)
if language and language not in entry['Code language'] and language not in ignored_languages:
entry['Code language'].append(osg_parse.ValueWithComment(language))
entry['Code language'].append(language)
print(' added to languages: {}'.format(language))
# contributors
@ -155,7 +155,7 @@ def github_import():
# look up author in entry developers
if name not in entry.get('Developer', []):
print(' dev "{}" added to entry {}'.format(name, file))
entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(name)]
entry['Developer'] = entry.get('Developer', []) + [name]
# look up author in developers data base
if name in all_developers:
@ -204,7 +204,7 @@ def github_starring_synchronization():
# get repos
code_repositories = entry.get('Code repository', [])
repos = [x.value for x in code_repositories if x.startswith(prefix)]
repos = [x for x in code_repositories if x.startswith(prefix)]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]

8
code/gitlab_import.py
View File

@ -54,7 +54,7 @@ def gitlab_import():
# read entry
entry = osg.read_entry(file)
code_repositories = entry['Code repository']
repos = [x.value for x in code_repositories if x.startswith(prefix)]
repos = [x for x in code_repositories if x.startswith(prefix)]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]
@ -77,7 +77,7 @@ def gitlab_import():
# search for repository
for r in code_repositories:
if r.value.startswith(repo):
if r.startswith(repo):
break
# update comment
@ -94,7 +94,7 @@ def gitlab_import():
# language in languages
for language, usage in info['languages'].items():
if language in c.known_languages and usage > 5 and language not in entry['Code language']:
entry['Code language'].append(osg_parse.ValueWithComment(language))
entry['Code language'].append(language)
print(' added to languages: {}'.format(language))
entry['Code repository'] = code_repositories
@ -125,7 +125,7 @@ def gitlab_starring_synchronization():
# get repos
code_repositories = entry.get('Code repository', [])
repos = [x.value for x in code_repositories if x.startswith(prefix)]
repos = [x for x in code_repositories if x.startswith(prefix)]
repos[0] += ' @add'
repos = [x for x in repos if '@add' in x]
repos = [x.split(' ')[0] for x in repos]

230
code/html/generate_static_website.py
View File

@ -80,7 +80,7 @@ import math
import datetime
import time
from functools import partial
from utils import osg, constants as c, utils
from utils import osg, constants as c, utils, osg_statistics as stat, osg_parse
from jinja2 import Environment, FileSystemLoader
import html5lib
@ -97,13 +97,13 @@ alphabet_replacements = {'Á': 'A', 'Å': 'A', 'Ä': 'A', 'É': 'E', 'ⴹ': 'E',
# the subfolder structure
games_path = ['games']
frameworks_path = ['frameworks']
non_games_path = ['frameworks']
inspirations_path = ['inspirations']
developers_path = ['developers']
# derived paths
games_index_path = games_path + ['index.html']
frameworks_index_path = frameworks_path + ['index.html']
non_games_index_path = non_games_path + ['index.html']
inspirations_index_path = inspirations_path + ['index.html']
developers_index_path = developers_path + ['index.html']
@ -155,11 +155,12 @@ genre_icon_map = {
# cross-references to the langue pages
code_language_references = {l: games_by_language_path[:-1] + ['{}#{}'.format(games_by_language_path[-1], osg.canonical_name(l))] for l in c.known_languages}
# map of internal framework names to display names (which are in plural)
framework_names = {
# map of internal non game names to display names (which are in plural)
non_game_category_names = {
'tool': 'Tools',
'framework': 'Frameworks',
'library': 'Libraries'
'library': 'Libraries',
'game engine': 'Game Engine'
}
# we check the output html structure every time
@ -284,15 +285,18 @@ def url_to(current, target, info=None):
return url
def preprocess(list, key, url):
def preprocess(items, key, url):
"""
Sets a few additional fields in the entries, inspirations, developers in order to generate the right content from
them later.
:param list:
:param url:
:param items:
:param key:
:return:
"""
_ = set()
for item in list:
_ = set() # this is just to avoid duplicating anchors
for item in items:
# add unique anchor ref
anchor = osg.canonical_name(item[key])
while anchor in _:
@ -311,6 +315,11 @@ def preprocess(list, key, url):
def game_index(entry):
"""
:param entry:
:return:
"""
e = {
'url': make_url(entry['href'], entry['Title']),
'anchor-id': entry['anchor-id']
@ -326,6 +335,11 @@ def game_index(entry):
def inspiration_index(inspiration):
"""
:param inspiration:
:return:
"""
e = {
'url': make_url(inspiration['href'], inspiration['Name']),
'anchor-id': inspiration['anchor-id'],
@ -337,6 +351,11 @@ def inspiration_index(inspiration):
def developer_index(developer):
"""
:param developer:
:return:
"""
e = {
'url': make_url(developer['href'], developer['Name']),
'anchor-id': developer['anchor-id']
@ -348,7 +367,12 @@ def developer_index(developer):
def shortcut_url(url, name):
"""
:param url:
:param name:
:return:
"""
# remove slash at the end
if url.endswith('/'):
url = url[:-1]
@ -388,6 +412,14 @@ def shortcut_url(url, name):
def make_url(href, content, title=None, css_class=None):
"""
:param href:
:param content:
:param title:
:param css_class:
:return:
"""
if isinstance(content, str):
content = make_text(content)
url = {
@ -403,9 +435,15 @@ def make_url(href, content, title=None, css_class=None):
def make_repo_url(x, name):
"""
:param x:
:param name:
:return:
"""
# parse comments
comments = []
if x.has_comment():
if isinstance(x, osg_parse.Value):
for c in x.comment.split(','):
c = c.strip()
if not c.startswith('@'):
@ -427,7 +465,7 @@ def make_repo_url(x, name):
else:
comments.append(make_icon('star-o', 'low rated'))
# this is the default element
url = make_url(x.value, shortcut_url(x.value, name))
url = make_url(x, shortcut_url(x, name))
if comments:
return make_enumeration([url, make_enclose(make_enumeration(comments), '(', ')')], '')
else:
@ -435,6 +473,12 @@ def make_repo_url(x, name):
def make_icon(css_class, title=None):
"""
:param css_class:
:param title:
:return:
"""
icon = {
'type': 'icon',
'class': css_class,
@ -445,6 +489,12 @@ def make_icon(css_class, title=None):
def make_text(content, css_class=None):
"""
:param content:
:param css_class:
:return:
"""
text = {
'type': 'text',
'text': content
@ -454,13 +504,14 @@ def make_text(content, css_class=None):
return text
def make_nothing():
return {
'type': 'nothing'
}
def make_enclose(entry, left, right):
"""
:param entry:
:param left:
:param right:
:return:
"""
enclose = {
'type': 'enclose',
'entry': entry,
@ -471,6 +522,12 @@ def make_enclose(entry, left, right):
def make_enumeration(entries, divider=', '):
"""
:param entries:
:param divider:
:return:
"""
enumeration = {
'type': 'enumeration',
'entries': entries,
@ -480,6 +537,11 @@ def make_enumeration(entries, divider=', '):
def make_tags(entries):
"""
:param entries:
:return:
"""
return {
'type': 'tags',
'enumeration': make_enumeration(entries, divider='')
@ -505,6 +567,12 @@ def developer_profile_link(link):
def convert_inspirations(inspirations, entries):
"""
:param inspirations:
:param entries:
:return:
"""
entries_references = {entry['Title']: entry['href'] for entry in entries}
for inspiration in inspirations:
name = inspiration['Name']
@ -524,6 +592,12 @@ def convert_inspirations(inspirations, entries):
def convert_developers(developers, entries):
"""
:param developers:
:param entries:
:return:
"""
entries_references = {entry['Title']:entry['href'] for entry in entries}
for developer in developers:
name = developer['Name']
@ -552,9 +626,14 @@ def convert_developers(developers, entries):
def create_keyword_tag(keyword):
"""
:param keyword:
:return:
"""
if keyword in c.recommended_keywords:
if keyword in c.non_game_keywords:
url = frameworks_index_path.copy()
url = non_games_index_path.copy()
else:
url = games_by_genres_path.copy()
url[-1] += '#{}'.format(keyword)
@ -567,6 +646,11 @@ def create_keyword_tag(keyword):
def create_state_texts(states):
"""
:param states:
:return:
"""
texts = []
if 'mature' in states:
texts.append(make_text('mature', 'has-text-weight-bold'))
@ -581,6 +665,13 @@ def create_state_texts(states):
def convert_entries(entries, inspirations, developers):
"""
:param entries:
:param inspirations:
:param developers:
:return:
"""
inspirations_references = {inspiration['Name']: inspiration['href'] for inspiration in inspirations}
developer_references = {developer['Name']: developer['href'] for developer in developers}
for entry in entries:
@ -596,7 +687,7 @@ def convert_entries(entries, inspirations, developers):
entry['note'] = make_text(entry['Note'], 'is-italic')
# keywords as tags
e = [create_keyword_tag(x.value) for x in entry['Keyword']]
e = [create_keyword_tag(x) for x in entry['Keyword']]
entry['keyword'] = make_tags(e)
# other normal fields (not technical info)
@ -604,8 +695,6 @@ def convert_entries(entries, inspirations, developers):
if field in entry:
e = entry[field]
divider = ', '
if isinstance(e[0], osg.osg_parse.ValueWithComment):
e = [x.value for x in e]
if field == 'Inspiration':
e = [make_url(inspirations_references[x], make_text(x)) for x in e]
elif field == 'Developer':
@ -627,8 +716,6 @@ def convert_entries(entries, inspirations, developers):
# platforms
if 'Platform' in entry:
e = entry['Platform']
if isinstance(e[0], osg.osg_parse.ValueWithComment):
e = [x.value for x in e]
else:
e = ['Unspecified']
e = [make_url(games_by_platform_path[:-1] + ['{}#{}'.format(games_by_platform_path[-1], x.lower())], make_icon(platform_icon_map[x]), x) if x in platform_icon_map else make_text(x) for x in e]
@ -643,8 +730,6 @@ def convert_entries(entries, inspirations, developers):
divider = ', '
if not e:
continue
if isinstance(e[0], osg.osg_parse.ValueWithComment) and field != 'Code repository':
e = [x.value for x in e]
if field == 'Code repository':
e = [make_repo_url(x, name) for x in e]
elif field == 'Code language':
@ -663,8 +748,6 @@ def convert_entries(entries, inspirations, developers):
if field in entry['Building']:
e = entry['Building'][field]
divider = ', '
if isinstance(e[0], osg.osg_parse.ValueWithComment):
e = [x.value for x in e]
e = [make_url(c.build_system_urls[x], x, css_class='is-size-7') if x in c.build_system_urls else make_text(x, 'is-size-7') for x in e]
namex = make_text('{}: '.format(field), 'is-size-7')
entry[field.lower()] = [namex, make_enumeration(e, divider)]
@ -673,19 +756,29 @@ def convert_entries(entries, inspirations, developers):
def add_license_links_to_entries(entries):
"""
:param entries:
:return:
"""
for entry in entries:
licenses = entry['Code license']
licenses = [(c.license_urls.get(license.value, ''), license.value) for license in licenses]
licenses = [(c.license_urls.get(license, ''), license) for license in licenses]
entry['Code license'] = licenses
def get_top50_games(games):
"""
:param games:
:return:
"""
top50_games = []
for game in games:
# get stars of repositories
stars = 0
for repo in game.get('Code repository', []):
if repo.has_comment():
if isinstance(repo, osg_parse.Value):
for c in repo.comment.split(','):
c = c.strip()
if not c.startswith('@'):
@ -707,25 +800,24 @@ def get_top50_games(games):
def generate(entries, inspirations, developers):
"""
:param entries:
:param inspirations:
:param developers:
Regenerates the whole static website given an already imported set of entries, inspirations and developers.
These datasets must be valid for each other, i.e. each inspiration listed in entries must also have an
entry in inspirations and the same holds for developers.
"""
# split entries in games and frameworks
games, frameworks = [], []
# split entries in games and non-games
games, non_games = [], []
for entry in entries:
(games, frameworks)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry)
(games, non_games)[any([keyword in entry['Keyword'] for keyword in c.non_game_keywords])].append(entry)
# preprocess
preprocess(games, 'Title', games_path)
preprocess(frameworks, 'Title', frameworks_path)
preprocess(non_games, 'Title', non_games_path)
# TODO preprocess doesn't set the urls for frameworks correctly fix here, do better later
for framework in frameworks:
keyword = [keyword for keyword in c.non_game_keywords if keyword in framework['Keyword']][0]
framework['href'] = frameworks_path + ['{}.html#{}'.format(keyword, framework['anchor-id'])]
entries = games + frameworks
for non_game in non_games:
keyword = [keyword for keyword in c.non_game_keywords if keyword in non_game['Keyword']][0]
non_game['href'] = non_games_path + ['{}.html#{}'.format(keyword, non_game['anchor-id'])]
entries = games + non_games
preprocess(inspirations, 'Name', inspirations_path)
preprocess(developers, 'Name', developers_path)
@ -733,7 +825,7 @@ def generate(entries, inspirations, developers):
convert_inspirations(inspirations, entries)
convert_developers(developers, entries)
convert_entries(games, inspirations, developers)
convert_entries(frameworks, inspirations, developers)
convert_entries(non_games, inspirations, developers)
# set external links up
add_license_links_to_entries(games)
@ -749,7 +841,7 @@ def generate(entries, inspirations, developers):
games_by_genre = sort_into_categories(games, genres, lambda item, category: category.lower() in item['Keyword'])
games_by_platform = sort_into_categories(entries, c.valid_platforms, lambda item, category: category in item.get('Platform', []), 'Unspecified')
games_by_language = sort_into_categories(entries, c.known_languages, lambda item, category: category in item['Code language'])
frameworks_by_type = sort_into_categories(frameworks, c.non_game_keywords, lambda item, category: category in item['Keyword'])
non_games_by_type = sort_into_categories(non_games, c.non_game_keywords, lambda item, category: category in item['Keyword'])
# extract top 50 Github stars games
top50_games = get_top50_games(games)
@ -783,19 +875,19 @@ def generate(entries, inspirations, developers):
# index.html
base['active_nav'] = 'index'
index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(frameworks))) }
index = {'subtitle': make_text('Contains information about {} open source games and {} frameworks/tools.'.format(len(games), len(non_games))) }
template = environment.get_template('index.jinja')
write(template.render(index=index), ['index.html'])
# contribute.html
# contribute page
base['active_nav'] = 'contribute'
template = environment.get_template('contribute.jinja')
write(template.render(), ['contribute.html'])
# statistics
# statistics page
base['active_nav'] = 'statistics'
# preparation
# statistics preparation
template = environment.get_template('statistics.jinja')
data = {
'title': 'Statistics',
@ -803,48 +895,42 @@ def generate(entries, inspirations, developers):
}
# build-systems
build_systems = []
field = 'Build system'
for entry in entries:
if field in entry['Building']:
build_systems.extend(entry['Building'][field])
build_systems = [x.value for x in build_systems]
unique_build_systems = set(build_systems)
unique_build_systems = [(l, build_systems.count(l)) for l in unique_build_systems]
unique_build_systems.sort(key=lambda x: str.casefold(x[0])) # first sort by name
unique_build_systems.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first)
build_systems_stat = stat.get_build_systems(entries)
# TODO replace all entries < 10 with "others"
# TODO make Pie chart out of it
section = {
'title': 'Build system',
'items': ['{} ({})'.format(*item) for item in unique_build_systems]
'items': ['{} ({})'.format(*item) for item in build_systems_stat]
}
data['sections'].append(section)
# render and write statistics page
write(template.render(data=data), ['statistics.html'])
# frameworks folder
base['url_to'] = partial(url_to, frameworks_path)
# non-games folder
base['url_to'] = partial(url_to, non_games_path)
base['active_nav'] = 'frameworks'
# frameworks by type
index = divide_in_three_columns_and_transform(frameworks_by_type, game_index)
# non-games by type
index = divide_in_three_columns_and_transform(non_games_by_type, game_index)
index['title'] = make_text('Open source game frameworks/tools')
index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(frameworks)))
index['subtitle'] = make_text('Index of {} game frameworks/tools'.format(len(non_games)))
index['categories'] = c.non_game_keywords
index['category-names'] = framework_names
index['category-names'] = non_game_category_names
index['category-icons'] = {}
index['number_entries_per_category_threshold'] = 0
index['entry_bold'] = lambda x: 'tags' not in x
index['category-infos'] = {}
write(template_categorical_index.render(index=index), frameworks_index_path)
write(template_categorical_index.render(index=index), non_games_index_path)
# generate frameworks pages
# generate non-games pages
for keyword in c.non_game_keywords:
listing = {
'title': framework_names[keyword],
'subtitle': make_url(frameworks_index_path, 'Index'),
'items': frameworks_by_type[keyword]
'title': non_game_category_names[keyword],
'subtitle': make_url(non_games_index_path, 'Index'),
'items': non_games_by_type[keyword]
}
write(template_listing_entries.render(listing=listing), frameworks_path +['{}.html'.format(keyword)])
write(template_listing_entries.render(listing=listing), non_games_path + ['{}.html'.format(keyword)])
# games folder
base['url_to'] = partial(url_to, games_path)

5
code/html/statistics.jinja
View File

@ -1,12 +1,13 @@
{% extends "base.jinja" %}
{% block content %}
<div class="container">
<p class="title is-4">{{ data['title'] }}</p>
<div class="container">
<div class="box"><p class="title is-4">{{ data['title'] }}</p></div>
{% set comma = joiner(",") %}
{% for section in data['sections'] %}
{{ comma() }} <a href="#">{{ section['title'] }}</a>
{% endfor %}
</div>
{#- each statistics section -#}
{% for section in data['sections'] %}
<section class="section">
<div class="container">

3
code/maintenance_developers.py
View File

@ -100,7 +100,6 @@ class DevelopersMaintainer:
entry_name = entry['Title']
entry_devs = entry.get('Developer', [])
for entry_dev in entry_devs:
entry_dev = entry_dev.value # ignore a possible comment
if entry_dev in self.developers:
self.developers[entry_dev]['Games'].append(entry_name)
else:
@ -127,7 +126,7 @@ class DevelopersMaintainer:
# for entry in self.entries:
# for developer in entry.get('Developer', []):
# if developer.comment:
# print('{:<25} - {:<25} - {}'.format(entry['File'], developer.value, developer.comment))
# print('{:<25} - {:<25} - {}'.format(entry['File'], developer, developer.comment))
if __name__ == "__main__":

72
code/maintenance_entries.py
View File

@ -56,7 +56,6 @@ def create_toc(title, file, entries):
rows = []
for entry in entries:
info = entry['Code language'] + entry['Code license'] + entry['State']
info = [x.value for x in info]
rows.append('- **[{}]({})** ({})'.format(entry['Title'], '../' + entry['File'], ', '.join(info)))
# sort rows (by title)
@ -130,7 +129,6 @@ class EntriesMaintainer:
keywords.extend(entry['Keyword'])
if b'first\xe2\x80\x90person'.decode() in entry['Keyword']:
print(entry['File'])
keywords = [x.value for x in keywords]
# reduce those starting with "multiplayer"
keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords]
@ -159,7 +157,6 @@ class EntriesMaintainer:
for entry in self.entries:
deps = entry.get('Code dependency', [])
for dependency in deps:
dependency = dependency.value
if dependency in referenced_dependencies:
referenced_dependencies[dependency] += 1
else:
@ -516,7 +513,6 @@ class EntriesMaintainer:
languages = []
for entry in self.entries:
languages.extend(entry[field])
languages = [x.value for x in languages]
unique_languages = set(languages)
unique_languages = [(l, languages.count(l) / len(languages)) for l in unique_languages]
@ -538,7 +534,6 @@ class EntriesMaintainer:
licenses = []
for entry in self.entries:
licenses.extend(entry[field])
licenses = [x.value for x in licenses]
unique_licenses = set(licenses)
unique_licenses = [(l, licenses.count(l) / len(licenses)) for l in unique_licenses]
@ -560,7 +555,6 @@ class EntriesMaintainer:
keywords = []
for entry in self.entries:
keywords.extend(entry[field])
keywords = [x.value for x in keywords]
# reduce those starting with "multiplayer"
keywords = [x if not x.startswith('multiplayer') else 'multiplayer' for x in keywords]
@ -597,7 +591,7 @@ class EntriesMaintainer:
popular = False
for repo in entry.get(field, []):
for popular_repo in popular_code_repositories:
if popular_repo in repo.value:
if popular_repo in repo:
popular = True
break
# if there were repositories, but none popular, add them to the list
@ -618,7 +612,6 @@ class EntriesMaintainer:
if field in entry:
code_dependencies.extend(entry[field])
entries_with_code_dependency += 1
code_dependencies = [x.value for x in code_dependencies]
statistics += 'With code dependency field {} ({:.1f}%)\n\n'.format(entries_with_code_dependency,
rel(entries_with_code_dependency))
@ -644,7 +637,6 @@ class EntriesMaintainer:
for entry in self.entries:
if field in entry['Building']:
build_systems.extend(entry['Building'][field])
build_systems = [x.value for x in build_systems]
statistics += 'Build systems information available for {:.1f}% of all projects.\n\n'.format(
rel(len(build_systems)))
@ -690,7 +682,6 @@ class EntriesMaintainer:
for entry in self.entries:
if field in entry:
platforms.extend(entry[field])
platforms = [x.value for x in platforms]
statistics += 'Platform information available for {:.1f}% of all projects.\n\n'.format(rel(len(platforms)))
@ -708,67 +699,12 @@ class EntriesMaintainer:
def update_html(self):
"""
Parses all entries, collects interesting info and stores it in a json file suitable for displaying
with a dynamic table in a browser.
"""
if not self.entries:
print('entries not yet loaded')
return
# make database out of it
db = {'headings': ['Game', 'Description', 'Download', 'State', 'Keyword', 'Source']}
entries = []
for info in self.entries:
# game & description
entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['Title'], info['Home'][0],
r'https://github.com/Trilarion/opensourcegames/blob/master/entries/' +
info['File']),
textwrap.shorten(info.get('Note', ''), width=60, placeholder='..')]
# download
field = 'Download'
if field in info and info[field]:
entry.append('<a href="{}">Link</a>'.format(info[field][0]))
else:
entry.append('')
# state (field state is essential)
entry.append('{} / {}'.format(info['State'][0],
'inactive since {}'.format(osg.extract_inactive_year(info)) if osg.is_inactive(info) else 'active'))
# keywords
keywords = info['Keyword']
keywords = [x.value for x in keywords]
entry.append(', '.join(keywords))
# source
text = []
field = 'Code repository'
if field in info and info[field]:
text.append('<a href="{}">Source</a>'.format(info[field][0].value))
languages = info['Code language']
languages = [x.value for x in languages]
text.append(', '.join(languages))
licenses = info['Code license']
licenses = [x.value for x in licenses]
text.append(', '.join(licenses))
entry.append(' - '.join(text))
# append to entries
entries.append(entry)
# sort entries by game name
entries.sort(key=lambda x: str.casefold(x[0]))
db['data'] = entries
# output
text = json.dumps(db, indent=1)
utils.write_text(c.json_db_file, text)
print('HTML updated')
print('HTML not updated')
def update_repos(self):
"""
@ -784,7 +720,6 @@ class EntriesMaintainer:
# for every entry filter those that are known git repositories (add additional repositories)
for entry in self.entries:
repos = entry.get('Code repository', [])
repos = [x.value for x in repos]
# keep the first and all others containing @add
if not repos:
continue
@ -841,7 +776,6 @@ class EntriesMaintainer:
git_repos = []
for entry in self.entries:
repos = entry['Code repository']
repos = [x.value for x in repos]
for repo in repos:
repo = repo.split(' ')[0].strip()
url = osg.git_repo(repo)
@ -898,7 +832,7 @@ class EntriesMaintainer:
# stats = {}
# for entry in self.entries:
# repos = entry.get('Code repository', [])
# comments = [x.comment for x in repos if x.value.startswith('https://github.com/') and x.comment]
# comments = [x.comment for x in repos if x.startswith('https://github.com/') and x.comment]
# for comment in comments:
# for part in comment.split(','):
# part = part.strip()

1
code/maintenance_inspirations.py
View File

@ -152,7 +152,6 @@ class InspirationMaintainer:
for entry in self.entries:
entry_name = entry['Title']
for inspiration in entry.get('Inspiration', []):
inspiration = inspiration.value
if inspiration in self.inspirations:
self.inspirations[inspiration]['Inspired entries'].append(entry_name)
else:

4
code/rejected.txt
View File

@ -79,6 +79,7 @@ HistoryLine (https://github.com/oliverdb/Historyline): Very early development, n
Homeworld SDL (https://github.com/aheadley/homeworld): Not open source compliant license (see https://github.com/aheadley/homeworld/blob/master/README)
Howitzer Skirmish (http://howski.sourceforge.net/): No source code
Ikariam (https://github.com/advocaite/ikariam, https://github.com/TheOnly92/Ikariem): No license information found, no assets license information found
Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files)
imperialism-remake (http://remake.twelvepm.de/, https://github.com/Trilarion/imperialism-Remake): Too minimal (I must know)
Imperium: Sticks (http://rtciv.sourceforge.net/, http://sourceforge.net/projects/rtciv): No source code available
ImperiumAO (https://sourceforge.net/projects/impao, https://www.imperiumao.com.ar/): Only engine is open source, engine is ORE
@ -209,5 +210,4 @@ XQuest 2 (http://www.swallowtail.org/xquest/, http://www.swallowtail.org/xquest/
xrick (http://www.bigorno.net/xrick): No open source license/unclear license (see file README in http://www.bigorno.net/xrick/xrick-021212.zip)
Yave (https://github.com/gan74/Yave): General graphics engine, not game centered in any way and experimental
Yuris Revenge (https://github.com/cookgreen/Yuris-Revenge): Mod to OpenRA
zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance
Imperia (http://imperiagame.wordpress.com/, https://sourceforge.net/projects/imperia/): No sources found, no clear license information found (old download at https://www.indiedb.com/games/imperia/downloads/imperia-alpha-v401b-setup-files)
zedragon (https://github.com/charlierobson/zedragon.git): License not found, Assembly, not sure which OS is supported, no release, not much guidance

18
code/synchronization/osgameclones_synchronization.py
View File

@ -281,14 +281,14 @@ if __name__ == "__main__":
osgc_languages = osgc_entry['lang']
if type(osgc_languages) == str:
osgc_languages = [osgc_languages]
our_languages = [x.value for x in our_entry['Code language']] # essential field
our_languages = our_entry['Code language']
p += compare_sets(osgc_languages, our_languages, 'code language')
# compare their license with our code and assets license
if 'license' in osgc_entry:
osgc_licenses = osgc_entry['license']
our_code_licenses = [x.value for x in our_entry['Code license']] # essential field
our_assets_licenses = [x.value for x in our_entry.get('Assets license', [])]
our_code_licenses = our_entry['Code license'] # essential field
our_assets_licenses = our_entry.get('Assets license', [])
p += compare_sets(osgc_licenses, our_code_licenses + our_assets_licenses, 'licenses', 'notthem')
p += compare_sets(osgc_licenses, our_code_licenses, 'licenses', 'notus')
@ -298,7 +298,7 @@ if __name__ == "__main__":
osgc_frameworks = osgc_entry['framework']
if type(osgc_frameworks) == str:
osgc_frameworks = [osgc_frameworks]
our_frameworks = [x.value for x in our_entry.get('Code dependency', [])]
our_frameworks = our_entry.get('Code dependency', [])
our_frameworks = [x.casefold() for x in our_frameworks]
our_frameworks = [x if x not in our_framework_replacements else our_framework_replacements[x] for x
in our_frameworks]
@ -315,13 +315,13 @@ if __name__ == "__main__":
'sourceforge.net/projects/')] # we don't need the general sites there
# osgc_repos = [x for x in osgc_repos if not x.startswith('https://sourceforge.net/projects/')] # ignore some
our_repos = our_entry.get('Code repository', [])
our_repos = [utils.strip_url(url.value) for url in our_repos]
our_repos = [utils.strip_url(url) for url in our_repos]
our_repos = [x for x in our_repos if not x.startswith(
'gitlab.com/osgames/')] # we do not yet spread our own deeds (but we will some day)
our_repos = [x for x in our_repos if
'cvs.sourceforge.net' not in x and 'svn.code.sf.net/p/' not in x] # no cvs or svn anymore
our_downloads = our_entry.get('Download', [])
our_downloads = [utils.strip_url(url.value) for url in our_downloads]
our_downloads = [utils.strip_url(url) for url in our_downloads]
p += compare_sets(osgc_repos, our_repos + our_downloads, 'repo',
'notthem') # if their repos are not in our downloads or repos
p += compare_sets(osgc_repos, our_repos[:1], 'repo',
@ -334,7 +334,7 @@ if __name__ == "__main__":
osgc_urls = [osgc_urls]
osgc_urls = [utils.strip_url(url) for url in osgc_urls]
our_urls = our_entry['Home']
our_urls = [utils.strip_url(url.value) for url in our_urls]
our_urls = [utils.strip_url(url) for url in our_urls]
p += compare_sets(osgc_urls, our_urls, 'url/home', 'notthem') # if their urls are not in our urls
# our_urls = [url for url in our_urls if
# not url.startswith('github.com/')] # they don't have them as url
@ -361,10 +361,10 @@ if __name__ == "__main__":
p += ' development : mismatch : them complete, us not mature\n'
# get our keywords
our_keywords = [x.value for x in our_entry['Keyword']] # essential
our_keywords = our_entry['Keyword'] # essential
# compare their originals to our inspirations
our_originals = [x.value for x in our_entry.get('Inspiration', [])]
our_originals = our_entry.get('Inspiration', [])
if 'originals' in osgc_entry:
osgc_originals = osgc_entry['originals']
osgc_originals = [x.replace(',', '') for x in

4
code/synchronization/sourceforge_import.py
View File

@ -76,7 +76,7 @@ def sourceforge_import():
# read full entry
entry = osg.read_entry(file)
developers = entry.get('Developer', [])
urls = [x.value for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')]
urls = [x for x in entry['Home'] if x.startswith('https://sourceforge.net/projects/')]
# do we need to save it again
entry_changed = False
@ -133,7 +133,7 @@ def sourceforge_import():
# look author up in entry developers field, if not existing add
if author_name not in developers:
print(' dev "{}" added to entry {}'.format(author_name, file))
entry['Developer'] = entry.get('Developer', []) + [osg_parse.ValueWithComment(author_name)]
entry['Developer'] = entry.get('Developer', []) + [author_name]
entry_changed = True
developers = entry.get('Developer', []) # update developers

5
code/utils/constants.py
View File

@ -68,9 +68,10 @@ fields_without_comments = ('Inspiration', 'Play', 'Download', 'Platform', 'Code
recommended_keywords = (
'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation',
'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
# TODO unmake remake a recommended keyword (should be the same as clone maybe), i.e. add another recommended keyword if only remake is in there
# non game keywords take precedence
non_game_keywords = ('framework', 'library', 'tool')
# non game keywords take precedence over other (game) recommended keywords, at most one of them per entry
non_game_keywords = ('framework', 'game engine', 'library', 'tool')
# known programming languages, anything else will result in a warning during a maintenance operation
# only these will be used when gathering statistics

53
code/utils/osg.py
View File

@ -340,11 +340,11 @@ def check_and_process_entry(entry):
if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
message += 'file name should be {}\n'.format(canonical_file_name)
# check that fields without comments have no comments, set to field without comment
# check that fields without comments have no comments (i.e. are no Values)
for field in c.fields_without_comments:
if field in entry:
content = entry[field]
if any(item.has_comment() for item in content):
if any(isinstance(item, osg_parse.Value) for item in content):
message += 'field without comments {} has comment\n'.format(field)
# state must contain either beta or mature but not both
@ -359,8 +359,8 @@ def check_and_process_entry(entry):
for field in c.url_fields:
values = entry.get(field, [])
for value in values:
if value.value.startswith('<') and value.value.endswith('>'):
value.value = value.value[1:-1]
if value.startswith('<') and value.endswith('>'):
value = value[1:-1]
if not any(value.startswith(x) for x in c.valid_url_prefixes):
message += 'URL "{}" in field "{}" does not start with a valid prefix'.format(value, field)
@ -368,7 +368,7 @@ def check_and_process_entry(entry):
for repo in entry.get('Code repository', []):
if any(repo.startswith(x) for x in ('@', '?')):
continue
repo = repo.value.split(' ')[0].strip()
repo = repo.split(' ')[0].strip()
if any((x in repo for x in ('github', 'gitlab', 'git.tuxfamily', 'git.savannah'))):
if not repo.startswith('https://'):
message += 'Repo "{}" should start with https://'.format(repo)
@ -420,7 +420,7 @@ def is_inactive(entry):
def extract_inactive_year(entry):
state = entry['State']
phrase = 'inactive since '
inactive_year = [x.value[len(phrase):] for x in state if x.startswith(phrase)]
inactive_year = [x[len(phrase):] for x in state if x.startswith(phrase)]
assert len(inactive_year) <= 1
if inactive_year:
return int(inactive_year[0])
@ -457,9 +457,29 @@ def write_entry(entry):
utils.write_text(entry_path, content)
def create_entry_content(entry):
def render_value(value):
"""
:param value:
:return:
"""
if isinstance(value, osg_parse.Value):
comment = value.comment
else:
comment = None
if any(x in value for x in (',', ' (')):
value = '"{}"'.format(value)
if comment:
return '{} ({})'.format(value, comment)
else:
return value
def create_entry_content(entry):
"""
Creates the entry content from an internal representation as dictionary with fields to a text file representation
that can be stored in the md files. It should be compatible with the gramar and reading a file and re-creating the
content should not change the content. Importanly, the comments of the values have to be added here.
:param entry:
:return:
"""
@ -468,7 +488,7 @@ def create_entry_content(entry):
content = '# {}\n\n'.format(entry['Title'])
# we automatically sort some fields
sort_fun = lambda x: str.casefold(x.value)
sort_fun = lambda x: str.casefold(x)
for field in ('Media', 'Inspiration', 'Code Language', 'Developer', 'Build system'):
if field in entry:
values = entry[field]
@ -479,12 +499,11 @@ def create_entry_content(entry):
b = [x for x in keywords if x not in c.recommended_keywords]
entry['Keyword'] = sorted(a, key=sort_fun) + sorted(b, key=sort_fun)
# now all properties in the recommended order
# now all properties are in the recommended order
for field in c.valid_properties:
if field in entry:
e = entry[field]
e = ['"{}"'.format(x) if any(y in x.value for y in (',', ' (')) else x for x in e]
e = [str(x) for x in e]
e = [render_value(x) for x in e]
e = list(dict.fromkeys(e)) # this removes duplicates while keeping the sorting order
content += '- {}: {}\n'.format(field, ', '.join(e))
content += '\n'
@ -504,8 +523,8 @@ def create_entry_content(entry):
has_properties = True
content += '\n'
e = entry['Building'][field]
e = ['"{}"'.format(x) if ',' in x else x for x in e]
e = [str(x) for x in e]
e = [render_value(x) for x in e]
e = list(dict.fromkeys(e)) # this removes duplicates while keeping the sorting order
content += '- {}: {}\n'.format(field, ', '.join(e))
# if there is a note, insert it
@ -533,16 +552,14 @@ def all_urls(entries):
:param entries:
:return:
"""
# TODO there are other fields than c.url_fields and also in comments, maybe just regex on the whole content
# TODO this might be part of the external link check or it might not, check for duplicate code
urls = {}
# iterate over entries
for entry in entries:
file = entry['File']
for field in c.url_fields: # TODO there are other fields, maybe just regex on the whole content
for field in c.url_fields:
for value in entry.get(field, []):
if value.comment:
value = value.value + ' ' + value.comment
else:
value = value.value
for subvalue in value.split(' '):
subvalue = subvalue.strip()
if is_url(subvalue):

68
code/utils/osg_parse.py
View File

@ -14,10 +14,10 @@ class ListingTransformer(lark.Transformer):
"""
def unquoted_value(self, x):
return x[0].value.strip()
return x[0].strip()
def quoted_value(self, x):
return x[0].value[1:-1].strip() # remove quotation marks and strip whitespaces
return x[0][1:-1].strip() # remove quotation marks and strip whitespaces
def property(self, x):
"""
@ -25,7 +25,7 @@ class ListingTransformer(lark.Transformer):
:param x:
:return:
"""
return x[0].value, x[1:]
return x[0], x[1:]
def name(self, x):
"""
@ -33,7 +33,7 @@ class ListingTransformer(lark.Transformer):
:param x:
:return:
"""
return 'Name', x[0].value.strip()
return 'Name', x[0].strip()
def entry(self, x):
"""
@ -56,19 +56,25 @@ class ListingTransformer(lark.Transformer):
class EntryTransformer(lark.Transformer):
def unquoted_value(self, x):
return x[0].value.strip()
return x[0].strip()
def quoted_value(self, x):
return x[0].value[1:-1].strip() # remove quotation marks
return x[0][1:-1].strip() # remove quotation marks
def comment_value(self, x):
return x[0].value[1:-1].strip() # remove parenthesis
return x[0][1:-1].strip() # remove parenthesis
def value(self, x):
"""
This also stores the comment if needed.
:param x:
:return:
"""
if len(x) == 1:
v = ValueWithComment(value=x[0])
v = x[0]
else:
v = ValueWithComment(value=x[0], comment=x[1])
v = Value(*x)
return v
def property(self, x):
@ -77,10 +83,10 @@ class EntryTransformer(lark.Transformer):
:param x:
:return:
"""
return x[0].value.strip(), x[1:]
return x[0].strip(), x[1:]
def title(self, x):
return 'Title', x[0].value.strip()
return 'Title', x[0].strip()
def note(self, x):
"""
@ -90,7 +96,7 @@ class EntryTransformer(lark.Transformer):
"""
if not x:
raise lark.Discard
return 'Note', ''.join((x.value for x in x))
return 'Note', ''.join(x)
def building(self, x):
return 'Building', x
@ -98,40 +104,16 @@ class EntryTransformer(lark.Transformer):
def start(self, x):
return x
# TODO turns out ValueWithComment does not really solve problem but actually creates even some, are there alternatives like inheriting from string?
class ValueWithComment:
class Value(str):
"""
All our property values can have (optional) comments. This is the class that represents them to us and implements
equality and 'in' operator functionality purely on the value.
A value is a string with some additional meta object (a comment) but mostly behaves as a string.
"""
def __init__(self, value, comment=None):
self.value = value
self.comment = comment
def is_empty(self):
return self.value == ''
def has_comment(self):
return self.comment is not None
def startswith(self, str):
return self.value.startswith(str)
def __contains__(self, item):
return item in self.value
def __eq__(self, other):
return self.value == other
def __repr__(self):
if self.comment:
return '{} ({})'.format(self.value, self.comment)
else:
return '{}'.format(self.value)
def __hash__(self):
return hash(self.value)
def __new__(cls, value, comment):
obj = str.__new__(cls, value)
obj.comment = comment
return obj
def parse(parser, transformer, content):
tree = parser.parse(content)

21
code/utils/osg_statistics.py
View File

@ -1,4 +1,23 @@
"""
Central place to calculate statistics about the entries. Used for updating the statistics.md file and the statistics page
of the website.
"""
"""
def get_build_systems(entries):
"""
Given a list of entries, calculates statistics about the used build systems and returns the statistics as
sorted list of elements (build-system-name, occurence).
"n/a" is used if no build system was specified
"""
build_systems = []
for entry in entries:
build_systems.extend(entry['Building'].get('Build system', ['n/a']))
unique_build_systems = set(build_systems)
build_systems_stat = [(l, build_systems.count(l)) for l in unique_build_systems]
build_systems_stat.sort(key=lambda x: str.casefold(x[0])) # first sort by name
build_systems_stat.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first)
return build_systems_stat

6
entries/freee.md
View File

@ -2,9 +2,9 @@
- Home: http://edkolis.com/freee
- Inspiration: Space Empires IV
- State: beta
- State: beta
- Download: https://github.com/ekolis/FrEee/releases
- Keyword: strategy, turn-based, space, content open
- Keyword: strategy, content open, space, turn-based
- Code repository: https://github.com/ekolis/FrEee.git
- Code language: C#
- Code license: CC-BY-NC-SA-2.0 (https://github.com/ekolis/FrEee/blob/master/FrEee.Assets/Licenses/license.txt)
@ -12,4 +12,4 @@
## Building
- Build system: Visual Studio
- Build system: Visual Studio