AnotherFoxGuy/opensourcegames
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

updates on developer information (parsing with lark)

Browse Source
This commit is contained in:
Trilarion
2020-02-03 15:50:24 +01:00
parent 10625a854a
commit eeb20a670a
66 changed files with 354 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
code/backlog.txt
View File

@ -6,6 +6,8 @@ http://circularstudios.com/
http://cyxdown.free.fr/bs/
http://cyxdown.free.fr/f2b/
http://dead-code.org/home/
https://github.com/restorer/gloomy-dungeons-2
https://github.com/MyreMylar/pygame_gui
http://e-adventure.e-ucm.es/login/index.php (games of eAdventure)
http://ethernet.wasted.ch/
http://evolonline.org/about
@ -222,6 +224,7 @@ https://github.com/Cortrah/SpaceOperaDesign, https://github.com/Cortrah/SpaceOpe
https://github.com/cping/LGame
https://github.com/cymonsgames/CymonsGames (collection)
https://github.com/DaanVanYperen/artemis-odb-contrib
https://github.com/David20321/7DFPS (http://www.wolfire.com/receiver, not open source, for rejected list)
https://github.com/DeflatedPickle/FAOSDance
https://github.com/delaford/game
https://github.com/Donerkebap13/DonerComponents
@ -336,6 +339,7 @@ https://github.com/skypjack/entt
https://github.com/smlinux/nexuiz
https://github.com/SPC-Some-Polish-Coders/PopHead
https://github.com/SPC-Some-Polish-Coders/PopHead/
https://github.com/Squalr/Squally
https://github.com/stackos/Viry3D
https://github.com/stolencatkarma/CataclysmLD
https://github.com/Suprcode/mir3-zircon

15
code/grammar_entries.lark Normal file
View File

@ -0,0 +1,15 @@
start: title description property+ _E note _E? building
title: "# " /(?! ).+(?<! )/ "\n" _E // not starting or ending with a space
description: "_" /(?! ).+(?<![ _])/ "_\n" _E // single line not ending with underscore
property: "- " _key ": " _value "\n" // a property on a single line "- key: value"
_key: /(?! ).+?(?=:)/ // key: everything until next ":"
_value: /.+(?<! )/ // everything until the end of the line
building: "## Building\n" _E property+ _E? note // the "building" section
note: /(?![\-#]).*\n/* // Unstructured text, not starting with - or #
_E: /^$\n/m // empty new line (filtered from tree)

17
code/grammar_listing.lark Normal file
View File

@ -0,0 +1,17 @@
start: header entries*
header: "# " name " (" number ")\n" _E
entries: "## " name " (" number ")\n" _E property+ _E
property: "- " _key ": " _value "\n"
_key: /(?! ).+?(?=:)(?<! )/ // key: everything until next ":", not beginning or ending with a space
_value: /.+(?<! )/ // everything until the end of the line, not ending with a space
name: /.+?(?= \()/ // developer name: everything until " ("
number: /[0-9]+/
COMMENT: /^\[comment\]: #.*$\n/m // [comment]: # xxx
_E: /^$\n/m // empty new line
%ignore COMMENT

6
code/libregamewiki_import.py
View File

@ -25,7 +25,7 @@ def download_lgw_content():
# parameters
base_url = 'https://libregamewiki.org'
destination_path = os.path.join(constants.root_path, 'tools', 'lgw-import')
destination_path = os.path.join(constants.root_path, 'code', 'lgw-import')
utils.recreate_directory(destination_path)
# read and process the base url (get all games and categories)
@ -64,7 +64,7 @@ def download_lgw_content():
def parse_lgw_content():
# paths
import_path = os.path.join(constants.root_path, 'tools', 'lgw-import')
import_path = os.path.join(constants.root_path, 'code', 'lgw-import')
entries_file = os.path.join(import_path, '_lgw.json')
# iterate over all imported files
@ -250,7 +250,7 @@ def ignore_nonnumbers(entries, fields):
def clean_lgw_content():
# paths
import_path = os.path.join(constants.root_path, 'tools', 'lgw-import')
import_path = os.path.join(constants.root_path, 'code', 'lgw-import')
entries_file = os.path.join(import_path, '_lgw.json')
cleaned_entries_file = os.path.join(import_path, '_lgw.cleaned.json')

2
code/libregamewiki_synchronization.py
View File

@ -71,7 +71,7 @@ if __name__ == "__main__":
maximal_newly_created_entries = 40
# paths
lgw_import_path = os.path.join(constants.root_path, 'tools', 'lgw-import')
lgw_import_path = os.path.join(constants.root_path, 'code', 'lgw-import')
lgw_entries_file = os.path.join(lgw_import_path, '_lgw.cleaned.json')
# import lgw import

34
code/maintenance.py
View File

@ -739,8 +739,8 @@ def export_primary_code_repositories_json(infos):
for domain in ('repo.or.cz', 'anongit.kde.org', 'bitbucket.org', 'git.code.sf.net', 'git.savannah', 'git.tuxfamily', 'github.com', 'gitlab.com', 'gitlab.com/osgames', 'gitlab.gnome.org'):
print('{} on {}'.format(sum(1 if domain in x else 0 for x in git_repos), domain))
# write them to tools/git
json_path = os.path.join(c.root_path, 'tools', 'archives.json')
# write them to code/git
json_path = os.path.join(c.root_path, 'code', 'archives.json')
text = json.dumps(primary_repos, indent=1)
utils.write_text(json_path, text)
@ -768,8 +768,8 @@ def export_git_code_repositories_json():
# sort them alphabetically (and remove duplicates)
urls.sort()
# write them to tools/git
json_path = os.path.join(c.root_path, 'tools', 'git_repositories.json')
# write them to code/git
json_path = os.path.join(c.root_path, 'code', 'git_repositories.json')
text = json.dumps(urls, indent=1)
utils.write_text(json_path, text)
@ -789,7 +789,7 @@ def sort_text_file(file, name):
def clean_backlog(stripped_game_urls):
# read backlog and split
file = os.path.join(c.root_path, 'tools', 'backlog.txt')
file = os.path.join(c.root_path, 'code', 'backlog.txt')
text = utils.read_text(file)
text = text.split('\n')
@ -809,7 +809,7 @@ def check_validity_backlog():
import requests
# read backlog and split
file = os.path.join(c.root_path, 'tools', 'backlog.txt')
file = os.path.join(c.root_path, 'code', 'backlog.txt')
text = utils.read_text(file)
urls = text.split('\n')
urls = [x.split(' ')[0] for x in urls]
@ -835,9 +835,6 @@ def update_inspirations(infos):
print('update inspirations')
inspirations_file = os.path.join(c.root_path, 'inspirations.md')
inspirations = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n# Inspirations\n\n'
# collect information
originals = {}
for info in infos:
@ -852,8 +849,9 @@ def update_inspirations(infos):
else:
originals[original] = [name]
# number of inspirations
inspirations += '{} games listed as inspirations.\n\n'.format(len(originals))
inspirations_file = os.path.join(c.root_path, 'inspirations.md')
inspirations = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
inspirations += '# Inspirations ({})\n\n'.format(len(originals)) # add number of inspirations
# iterate through originals alphabetically sorted
for original, names in sorted(originals.items(), key=lambda x: str.casefold(x[0])):
@ -871,9 +869,6 @@ def update_developer(infos):
print('update developer')
developer_file = os.path.join(c.root_path, 'developer.md')
content = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n# Developer\n\n'
# collect information
developer = {}
for info in infos:
@ -886,8 +881,9 @@ def update_developer(infos):
else:
developer[dev] = [name]
# number of developers
content += '{} listed developer(s).\n\n'.format(len(developer))
developer_file = os.path.join(c.root_path, 'developer.md')
content = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
content += '# Developer ({})\n\n'.format(len(developer)) # add number of developer
# iterate through developers alphabetically sorted
for dev, names in sorted(developer.items(), key=lambda x: str.casefold(x[0])):
@ -936,7 +932,7 @@ if __name__ == "__main__":
# backlog
game_urls = osg.extract_links()
text = utils.read_text(os.path.join(c.root_path, 'tools', 'rejected.txt'))
text = utils.read_text(os.path.join(c.root_path, 'code', 'rejected.txt'))
regex = re.compile(r"\((http.*?)\)", re.MULTILINE)
matches = regex.findall(text)
rejected_urls = []
@ -992,5 +988,5 @@ if __name__ == "__main__":
# check_validity_external_links()
# sort backlog and rejected
# sort_text_file(os.path.join(c.root_path, 'tools', 'backlog.txt'), 'backlog')
sort_text_file(os.path.join(c.root_path, 'tools', 'rejected.txt'), 'rejected games list')
# sort_text_file(os.path.join(c.root_path, 'code', 'backlog.txt'), 'backlog')
sort_text_file(os.path.join(c.root_path, 'code', 'rejected.txt'), 'rejected games list')

2
code/osgameclones_synchronization.py
View File

@ -471,7 +471,7 @@ if __name__ == "__main__":
for our_entry in our_entries:
our_name = our_entry['name']
# only if contains a keyword starting with "inspired by" and not "tools", "framework" or "library"
# only if contains a keyword starting with "inspired by" and not "tool", "framework" or "library"
our_keywords = our_entry['keywords']
if not any([x.startswith('inspired by ') for x in our_keywords]):
continue

2
code/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
pygithub
lark-parser

1
code/utils/constants.py
View File

@ -8,5 +8,6 @@ import os
root_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
entries_path = os.path.join(root_path, 'entries')
tocs_path = os.path.join(entries_path, 'tocs')
code_path = os.path.join(root_path, 'code')
local_properties_file = os.path.join(root_path, 'local.properties')

168
code/utils/osg.py
View File

@ -6,24 +6,73 @@ import re
import os
from difflib import SequenceMatcher
from utils import utils, constants as c
import lark
class ListingTransformer(lark.Transformer):
def number(self, x):
raise lark.Discard
def property(self, x):
return (x[0].value, x[1].value)
def name(self, x):
return ('name', x[0].value)
def entries(self, x):
d = {}
for key, value in x:
d[key] = value
return d
def header(self, x):
raise lark.Discard
def start(self, x):
return x
essential_fields = ('Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
valid_fields = ('Home', 'Media', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Build system', 'Build instructions')
valid_fields = (
'Home', 'Media', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Build system', 'Build instructions')
valid_platforms = ('Windows', 'Linux', 'macOS', 'Android', 'iOS', 'Web')
recommended_keywords = ('action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation', 'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
known_languages = ('AGS Script', 'ActionScript', 'Ada', 'AngelScript', 'Assembly', 'Basic', 'Blender Script', 'BlitzMax', 'C', 'C#', 'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript', 'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua', 'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R', "Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala', 'Visual Basic', 'XUL', 'ZenScript', 'ooc')
known_licenses = ('2-clause BSD', '3-clause BSD', 'AFL-3.0', 'AGPL-3.0', 'Apache-2.0', 'Artistic License-1.0', 'Artistic License-2.0', 'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0', 'CC-BY-SA-4.0', 'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0', 'LGPL-2.1', 'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License', 'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
known_multiplayer_modes = ('competitive', 'co-op', 'hotseat', 'LAN', 'local', 'massive', 'matchmaking', 'online', 'split-screen')
recommended_keywords = (
'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation',
'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
known_languages = (
'AGS Script', 'ActionScript', 'Ada', 'AngelScript', 'Assembly', 'Basic', 'Blender Script', 'BlitzMax', 'C', 'C#',
'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript',
'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua',
'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R',
"Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
'Visual Basic', 'XUL', 'ZenScript', 'ooc')
known_licenses = (
'2-clause BSD', '3-clause BSD', 'AFL-3.0', 'AGPL-3.0', 'Apache-2.0', 'Artistic License-1.0', 'Artistic License-2.0',
'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0',
'CC-BY-SA-4.0',
'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0', 'LGPL-2.1',
'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License', 'None',
'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
known_multiplayer_modes = (
'competitive', 'co-op', 'hotseat', 'LAN', 'local', 'massive', 'matchmaking', 'online', 'split-screen')
# TODO put the abbreviations directly in the name line (parenthesis maybe), that is more natural
code_dependencies_aliases = {'Simple DirectMedia Layer': ('SDL', 'SDL2'), 'Simple and Fast Multimedia Library': 'SFML', 'Boost (C++ Libraries)': 'Boost', 'SGE Game Engine': 'SGE'}
code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/', 'GLUT': 'https://www.opengl.org/resources/libraries/', 'WebGL': 'https://www.khronos.org/webgl/', 'Unity': 'https://unity.com/solutions/game',
'.NET': 'https://dotnet.microsoft.com/', 'Vulkan': 'https://www.khronos.org/vulkan/', 'KDE Frameworks': 'https://kde.org/products/frameworks/'}
code_dependencies_aliases = {'Simple DirectMedia Layer': ('SDL', 'SDL2'), 'Simple and Fast Multimedia Library': 'SFML',
'Boost (C++ Libraries)': 'Boost', 'SGE Game Engine': 'SGE'}
code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
'GLUT': 'https://www.opengl.org/resources/libraries/',
'WebGL': 'https://www.khronos.org/webgl/',
'Unity': 'https://unity.com/solutions/game',
'.NET': 'https://dotnet.microsoft.com/', 'Vulkan': 'https://www.khronos.org/vulkan/',
'KDE Frameworks': 'https://kde.org/products/frameworks/'}
regex_sanitize_name = re.compile(r"[^A-Za-z 0-9-+]+")
regex_sanitize_name_space_eater = re.compile(r" +")
comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
def name_similarity(a, b):
return SequenceMatcher(None, str.casefold(a), str.casefold(b)).ratio()
@ -87,26 +136,27 @@ def parse_entry(content):
info = {}
# read name
regex = re.compile(r"^# (.*)") # start of content, starting with "# " and then everything until the end of line
regex = re.compile(r"^# (.*)") # start of content, starting with "# " and then everything until the end of line
matches = regex.findall(content)
if len(matches) != 1 or not matches[0]: # name must be there
if len(matches) != 1 or not matches[0]: # name must be there
raise RuntimeError('Name not found in entry "{}" : {}'.format(content, matches))
info['name'] = matches[0]
# read description
regex = re.compile(r"^.*\n\n_(.*)_\n") # third line from top, everything between underscores
regex = re.compile(r"^.*\n\n_(.*)_\n") # third line from top, everything between underscores
matches = regex.findall(content)
if len(matches) != 1 or not matches[0]: # description must be there
if len(matches) != 1 or not matches[0]: # description must be there
raise RuntimeError('Description not found in entry "{}"'.format(content))
info['description'] = matches[0]
# first read all field names
regex = re.compile(r"^- (.*?): ", re.MULTILINE) # start of each line having "- ", then everything until a colon, then ": "
regex = re.compile(r"^- (.*?): ",
re.MULTILINE) # start of each line having "- ", then everything until a colon, then ": "
fields = regex.findall(content)
# check that essential fields are there
for field in essential_fields:
if field not in fields: # essential fields must be there
if field not in fields: # essential fields must be there
raise RuntimeError('Essential field "{}" missing in entry "{}"'.format(field, info['name']))
# check that all fields are valid fields and are existing in that order
@ -114,19 +164,20 @@ def parse_entry(content):
for field in fields:
while index < len(valid_fields) and field != valid_fields[index]:
index += 1
if index == len(valid_fields): # must be valid fields and must be in the right order
raise RuntimeError('Field "{}" in entry "{}" either misspelled or in wrong order'.format(field, info['name']))
if index == len(valid_fields): # must be valid fields and must be in the right order
raise RuntimeError(
'Field "{}" in entry "{}" either misspelled or in wrong order'.format(field, info['name']))
# iterate over found fields
for field in fields:
regex = re.compile(r"- {}: (.*)".format(field))
matches = regex.findall(content)
if len(matches) != 1: # every field must be present only once
if len(matches) != 1: # every field must be present only once
raise RuntimeError('Field "{}" in entry "{}" exist multiple times.'.format(field, info['name']))
v = matches[0]
# first store as is
info[field.lower()+'-raw'] = v
info[field.lower() + '-raw'] = v
# remove parenthesis with content
v = re.sub(r'\([^)]*\)', '', v)
@ -152,7 +203,7 @@ def parse_entry(content):
# check again that essential fields made it through
for field in ('home', 'state', 'keywords', 'code language', 'code license'):
if field not in info: # essential fields must still be inside
if field not in info: # essential fields must still be inside
raise RuntimeError('Essential field "{}" empty in entry "{}"'.format(field, info['name']))
# now checks on the content of fields
@ -160,7 +211,7 @@ def parse_entry(content):
# name and description should not have spaces at the begin or end
for field in ('name', 'description'):
v = info[field]
if len(v) != len(v.strip()): # warning about that
if len(v) != len(v.strip()): # warning about that
print('Warning: No leading or trailing spaces in field {} in entry "{}"'.format(field, info['name']))
# state (essential field) must contain either beta or mature but not both, but at least one
@ -182,8 +233,11 @@ def parse_entry(content):
for field in ['home', 'download', 'play', 'code repository']:
if field in info:
for url in info[field]:
if not any([url.startswith(x) for x in ['http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://']]):
raise RuntimeError('URL "{}" in entry "{}" does not start with http/https/git/svn/ftp/bzr'.format(url, info['name']))
if not any(
[url.startswith(x) for x in ['http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://']]):
raise RuntimeError(
'URL "{}" in entry "{}" does not start with http/https/git/svn/ftp/bzr'.format(url,
info['name']))
if ' ' in url:
raise RuntimeError('URL "{}" in entry "{}" contains a space'.format(url, info['name']))
@ -202,8 +256,10 @@ def parse_entry(content):
for platform in info['platform']:
while index < len(valid_platforms) and platform != valid_platforms[index]:
index += 1
if index == len(valid_platforms): # must be valid platforms and must be in that order
raise RuntimeError('Platform tag "{}" in entry "{}" either misspelled or in wrong order'.format(platform, info['name']))
if index == len(valid_platforms): # must be valid platforms and must be in that order
raise RuntimeError(
'Platform tag "{}" in entry "{}" either misspelled or in wrong order'.format(platform,
info['name']))
# there must be at least one keyword
if 'keywords' not in info:
@ -215,20 +271,22 @@ def parse_entry(content):
if recommended_keyword in info['keywords']:
fail = False
break
if fail: # must be at least one recommended keyword
if fail: # must be at least one recommended keyword
raise RuntimeError('Entry "{}" contains no recommended keyword'.format(info['name']))
# languages should be known
languages = info['code language']
for language in languages:
if language not in known_languages:
print('Warning: Language {} in entry "{}" is not a known language. Misspelled or new?'.format(language, info['name']))
print('Warning: Language {} in entry "{}" is not a known language. Misspelled or new?'.format(language,
info['name']))
# licenses should be known
licenses = info['code license']
for license in licenses:
if license not in known_licenses:
print('Warning: License {} in entry "{}" is not a known license. Misspelled or new?'.format(license, info['name']))
print('Warning: License {} in entry "{}" is not a known license. Misspelled or new?'.format(license,
info['name']))
return info
@ -295,3 +353,57 @@ def extract_links():
urls.add(url)
urls = sorted(list(urls), key=str.casefold)
return urls
def read_and_parse(content_file, grammar_file, transformer):
"""
:param content_file:
:param grammar_file:
:param transformer:
:return:
"""
content = utils.read_text(content_file)
grammar = utils.read_text(grammar_file)
parser = lark.Lark(grammar, debug=False)
tree = parser.parse(content)
return transformer.transform(tree)
def read_developer_info():
"""
:return:
"""
developer_file = os.path.join(c.root_path, 'developer.md')
grammar_file = os.path.join(c.code_path, 'grammar_listing.lark')
transformer = ListingTransformer()
return read_and_parse(developer_file, grammar_file, transformer)
def write_developer_info(developers):
"""
:return:
"""
developer_file = os.path.join(c.root_path, 'developer.md')
def read_inspirations_info():
"""
:return:
"""
inspirations_file = os.path.join(c.root_path, 'inspirations.md')
grammar_file = os.path.join(c.code_path, 'grammar_listing.lark')
transformer = ListingTransformer()
return read_and_parse(inspirations_file, grammar_file, transformer)
def write_inspirations_info(inspirations):
"""
:param inspirations:
:return:
"""
inspirations_file = os.path.join(c.root_path, 'inspirations.md')