entries cleanup
This commit is contained in:
Trilarion
@ -7,6 +7,7 @@ https://github.com/SadConsole/SadConsole
|
||||
https://github.com/tlgkccampbell/ultraviolet
|
||||
https://github.com/amerkoleci/Vortice.Windows
|
||||
https://github.com/horde3d/Horde3D
|
||||
https://github.com/delaford/game
|
||||
https://github.com/cxong/cdogs-sdl
|
||||
https://moaiwebsite.github.io/
|
||||
http://cyxdown.free.fr/bs/
|
||||
|
||||
@ -18,118 +18,47 @@ import re
|
||||
import utils.constants
|
||||
from utils import constants as c, utils, osg
|
||||
|
||||
|
||||
def update_readme_and_tocs(infos):
|
||||
def extract_links():
|
||||
"""
|
||||
Recounts entries in sub categories and writes them to the readme.
|
||||
Also updates the _toc files in the categories directories.
|
||||
|
||||
Note: The Readme must have a specific structure at the beginning, starting with "# Open Source Games" and ending
|
||||
on "A collection.."
|
||||
|
||||
Needs to be performed regularly.
|
||||
"""
|
||||
print('update readme and toc files')
|
||||
|
||||
# completely delete content of toc path
|
||||
for file in os.listdir(c.tocs_path):
|
||||
os.remove(os.path.join(c.tocs_path, file))
|
||||
|
||||
# read readme
|
||||
readme_file = os.path.join(c.root_path, 'README.md')
|
||||
readme_text = utils.read_text(readme_file)
|
||||
|
||||
# compile regex for identifying the building blocks in the readme
|
||||
regex = re.compile(r"(.*?)(\[comment\]: # \(start.*?end of autogenerated content\))(.*)", re.DOTALL)
|
||||
|
||||
# apply regex
|
||||
matches = regex.findall(readme_text)
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError('readme file has invalid structure')
|
||||
matches = matches[0]
|
||||
start = matches[0]
|
||||
end = matches[2]
|
||||
|
||||
tocs_text = ''
|
||||
|
||||
# split infos
|
||||
infos_games, infos_tools, infos_frameworks, infos_libraries = osg.split_infos(infos)
|
||||
|
||||
# create games, tools, frameworks, libraries tocs
|
||||
title = 'Games'
|
||||
file = '_games.md'
|
||||
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_games))
|
||||
create_toc(title, file, infos_games)
|
||||
|
||||
title = 'Tools'
|
||||
file = '_tools.md'
|
||||
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_tools))
|
||||
create_toc(title, file, infos_tools)
|
||||
|
||||
title = 'Frameworks'
|
||||
file = '_frameworks.md'
|
||||
tocs_text += '**[{}](entries/tocs/{}#{})** ({}) - '.format(title, file, title, len(infos_frameworks))
|
||||
create_toc(title, file, infos_frameworks)
|
||||
|
||||
title = 'Libraries'
|
||||
file = '_libraries.md'
|
||||
tocs_text += '**[{}](entries/tocs/{}#{})** ({})\n'.format(title, file, title, len(infos_libraries))
|
||||
create_toc(title, file, infos_libraries)
|
||||
|
||||
# create by category
|
||||
categories_text = []
|
||||
for keyword in utils.constants.recommended_keywords:
|
||||
infos_filtered = [x for x in infos if keyword in x['keywords']]
|
||||
title = keyword.capitalize()
|
||||
name = keyword.replace(' ', '-')
|
||||
file = '_{}.md'.format(name)
|
||||
categories_text.append('**[{}](entries/tocs/{}#{})** ({})'.format(title, file, name, len(infos_filtered)))
|
||||
create_toc(title, file, infos_filtered)
|
||||
categories_text.sort()
|
||||
tocs_text += '\nBy category: {}\n'.format(', '.join(categories_text))
|
||||
|
||||
# create by platform
|
||||
platforms_text = []
|
||||
for platform in utils.constants.valid_platforms:
|
||||
infos_filtered = [x for x in infos if platform in x.get('platform', [])]
|
||||
title = platform
|
||||
name = platform.lower()
|
||||
file = '_{}.md'.format(name)
|
||||
platforms_text.append('**[{}](entries/tocs/{}#{})** ({})'.format(title, file, name, len(infos_filtered)))
|
||||
create_toc(title, file, infos_filtered)
|
||||
tocs_text += '\nBy platform: {}\n'.format(', '.join(platforms_text))
|
||||
|
||||
# insert new text in the middle (the \n before the second comment is necessary, otherwise Markdown displays it as part of the bullet list)
|
||||
text = start + "[comment]: # (start of autogenerated content, do not edit)\n" + tocs_text + "\n[comment]: # (end of autogenerated content)" + end
|
||||
|
||||
# write to readme
|
||||
utils.write_text(readme_file, text)
|
||||
|
||||
|
||||
def create_toc(title, file, entries):
|
||||
Parses all entries and extracts http(s) links from them
|
||||
"""
|
||||
|
||||
# regex for finding urls (can be in <> or in ]() or after a whitespace
|
||||
regex = re.compile(r"[\s\n]<(http.+?)>|]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n,]")
|
||||
|
||||
# iterate over all entries
|
||||
urls = set()
|
||||
for _, _, content in entry_iterator():
|
||||
|
||||
# apply regex
|
||||
matches = regex.findall(content)
|
||||
|
||||
# for each match
|
||||
for match in matches:
|
||||
|
||||
# for each possible clause
|
||||
for url in match:
|
||||
|
||||
# if there was something (and not a sourceforge git url)
|
||||
if url:
|
||||
urls.add(url)
|
||||
urls = sorted(list(urls), key=str.casefold)
|
||||
return urls
|
||||
|
||||
def split_infos(infos):
|
||||
"""
|
||||
# file path
|
||||
toc_file = os.path.join(c.tocs_path, file)
|
||||
Split into games, tools, frameworks, libraries
|
||||
"""
|
||||
games = [x for x in infos if not any([y in x['keywords'] for y in ('tool', 'framework', 'library')])]
|
||||
tools = [x for x in infos if 'tool' in x['keywords']]
|
||||
frameworks = [x for x in infos if 'framework' in x['keywords']]
|
||||
libraries = [x for x in infos if 'library' in x['keywords']]
|
||||
return games, tools, frameworks, libraries
|
||||
|
||||
# header line
|
||||
text = '[comment]: # (autogenerated content, do not edit)\n# {}\n\n'.format(title)
|
||||
|
||||
# assemble rows
|
||||
rows = []
|
||||
for entry in entries:
|
||||
rows.append('- **[{}]({})** ({})'.format(entry['Name'], '../' + entry['file'], ', '.join(
|
||||
entry['code language'] + entry['code license'] + entry['state'])))
|
||||
|
||||
# sort rows (by title)
|
||||
rows.sort(key=str.casefold)
|
||||
|
||||
# add to text
|
||||
text += '\n'.join(rows)
|
||||
|
||||
# write to toc file
|
||||
utils.write_text(toc_file, text)
|
||||
|
||||
|
||||
def check_validity_external_links():
|
||||
@ -220,28 +149,6 @@ def check_validity_external_links():
|
||||
print('{}: {} - exception {}'.format(names, url, error_name))
|
||||
|
||||
|
||||
def check_template_leftovers():
|
||||
"""
|
||||
Checks for template leftovers.
|
||||
|
||||
Should be run only occasionally.
|
||||
"""
|
||||
|
||||
print('check for template leftovers')
|
||||
|
||||
# load template and get all lines
|
||||
text = utils.read_text(os.path.join(c.root_path, 'template.md'))
|
||||
text = text.split('\n')
|
||||
check_strings = [x for x in text if x and not x.startswith('##')]
|
||||
|
||||
# iterate over all entries
|
||||
for _, entry_path, content in osg.entry_iterator():
|
||||
|
||||
for check_string in check_strings:
|
||||
if content.find(check_string) >= 0:
|
||||
raise RuntimeError('{}: found {}'.format(os.path.basename(entry_path), check_string))
|
||||
|
||||
|
||||
def fix_entries():
|
||||
"""
|
||||
Fixes the keywords, code dependencies, build systems, .. entries, mostly by automatically sorting them.
|
||||
@ -879,72 +786,6 @@ def check_validity_backlog():
|
||||
print('{} redirected to {}, {}'.format(url, r.url, r.history))
|
||||
|
||||
|
||||
def update_inspirations(infos):
|
||||
"""
|
||||
|
||||
"""
|
||||
|
||||
print('update inspirations')
|
||||
|
||||
# collect information
|
||||
originals = {}
|
||||
for info in infos:
|
||||
name = info['Name']
|
||||
keywords = info['keywords']
|
||||
ins = [x[12:] for x in keywords if x.startswith('inspired by ')]
|
||||
if ins:
|
||||
ins = ins[0].split(' + ')
|
||||
for original in ins:
|
||||
if original in originals:
|
||||
originals[original].append(name)
|
||||
else:
|
||||
originals[original] = [name]
|
||||
|
||||
inspirations_file = os.path.join(c.root_path, 'inspirations.md')
|
||||
inspirations = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
|
||||
inspirations += '# Inspirations ({})\n\n'.format(len(originals)) # add number of inspirations
|
||||
|
||||
# iterate through originals alphabetically sorted
|
||||
for original, names in sorted(originals.items(), key=lambda x: str.casefold(x[0])):
|
||||
inspirations += '## {} ({})\n\n'.format(original, len(names))
|
||||
inspirations += '- Inspired entries: {}\n\n'.format(', '.join(sorted(names, key=str.casefold)))
|
||||
|
||||
# write to statistics file
|
||||
utils.write_text(inspirations_file, inspirations)
|
||||
|
||||
|
||||
def update_developer(infos):
|
||||
"""
|
||||
|
||||
"""
|
||||
|
||||
print('update developer')
|
||||
|
||||
# collect information
|
||||
developer = {}
|
||||
for info in infos:
|
||||
if 'developer' in info:
|
||||
name = info['Name']
|
||||
devs = info['developer']
|
||||
for dev in devs:
|
||||
if dev in developer:
|
||||
developer[dev].append(name)
|
||||
else:
|
||||
developer[dev] = [name]
|
||||
|
||||
developer_file = os.path.join(c.root_path, 'developer.md')
|
||||
content = '[comment]: # (partly autogenerated content, edit with care, read the manual before)\n'
|
||||
content += '# Developer ({})\n\n'.format(len(developer)) # add number of developer
|
||||
|
||||
# iterate through developers alphabetically sorted
|
||||
for dev, names in sorted(developer.items(), key=lambda x: str.casefold(x[0])):
|
||||
content += '## {} ({})\n\n'.format(dev, len(names))
|
||||
content += '- Games: {}\n\n'.format(', '.join(sorted(names, key=str.casefold)))
|
||||
|
||||
# write to statistics file
|
||||
utils.write_text(developer_file, content)
|
||||
|
||||
|
||||
def check_code_dependencies(infos):
|
||||
"""
|
||||
|
||||
@ -984,50 +825,20 @@ def check_code_dependencies(infos):
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# check_validity_backlog()
|
||||
check_validity_backlog()
|
||||
|
||||
|
||||
# clean backlog
|
||||
game_urls = osg.extract_links()
|
||||
text = utils.read_text(os.path.join(c.root_path, 'code', 'rejected.txt'))
|
||||
regex = re.compile(r"\((http.*?)\)", re.MULTILINE)
|
||||
matches = regex.findall(text)
|
||||
rejected_urls = []
|
||||
for match in matches:
|
||||
urls = match.split(',')
|
||||
urls = [x.strip() for x in urls]
|
||||
rejected_urls.extend(urls)
|
||||
game_urls.extend(rejected_urls)
|
||||
more_urls = []
|
||||
for url in game_urls:
|
||||
if url.startswith('https://web.archive.org/web'):
|
||||
# print(url) # sometimes the http is missing in archive links (would need proper parsing)
|
||||
url = url[url.index('http', 5):]
|
||||
more_urls.append(url)
|
||||
game_urls.extend(more_urls)
|
||||
stripped_game_urls = [utils.strip_url(x) for x in game_urls]
|
||||
clean_backlog(stripped_game_urls)
|
||||
|
||||
# check for unfilled template lines
|
||||
check_template_leftovers()
|
||||
|
||||
# fix entries
|
||||
fix_entries()
|
||||
|
||||
# assemble info
|
||||
infos = osg.assemble_infos()
|
||||
|
||||
# recount and write to readme and to tocs
|
||||
update_readme_and_tocs(infos)
|
||||
|
||||
# generate report
|
||||
update_statistics(infos)
|
||||
|
||||
# update inspirations
|
||||
# update_inspirations(infos)
|
||||
|
||||
# update developers
|
||||
# update_developer(infos)
|
||||
|
||||
# update database for html table
|
||||
export_json(infos)
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ valid_fields = ('File', 'Title') + valid_properties + ('Note', 'Building')
|
||||
|
||||
url_fields = ('Home', 'Media', 'Play', 'Download', 'Code repository')
|
||||
|
||||
valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://', '@see-')
|
||||
valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://', '@see-', '@not-', '?')
|
||||
|
||||
valid_building_properties = ('Build system', 'Build instructions')
|
||||
valid_building_fields = valid_building_properties + ('Note',)
|
||||
@ -63,8 +63,8 @@ known_languages = (
|
||||
'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript',
|
||||
'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua',
|
||||
'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R',
|
||||
"Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
|
||||
'Visual Basic', 'XUL', 'ZenScript', 'ooc')
|
||||
"Ren'Py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
|
||||
'Visual Basic', 'XUL', 'ZenScript', 'ooc', '?')
|
||||
|
||||
# known licenses, anything outside of this will result in a warning during a maintenance operation
|
||||
# only these will be used when gathering statistics
|
||||
@ -73,7 +73,7 @@ known_licenses = (
|
||||
'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0',
|
||||
'CC-BY-SA-4.0', 'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0',
|
||||
'LGPL-2.1', 'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License',
|
||||
'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
|
||||
'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib', '?')
|
||||
|
||||
# valid multiplayer modes (can be combined with "+" )
|
||||
valid_multiplayer_modes = (
|
||||
|
||||
@ -15,17 +15,6 @@ def name_similarity(a, b):
|
||||
return SequenceMatcher(None, str.casefold(a), str.casefold(b)).ratio()
|
||||
|
||||
|
||||
def split_infos(infos):
|
||||
"""
|
||||
Split into games, tools, frameworks, libraries
|
||||
"""
|
||||
games = [x for x in infos if not any([y in x['keywords'] for y in ('tool', 'framework', 'library')])]
|
||||
tools = [x for x in infos if 'tool' in x['keywords']]
|
||||
frameworks = [x for x in infos if 'framework' in x['keywords']]
|
||||
libraries = [x for x in infos if 'library' in x['keywords']]
|
||||
return games, tools, frameworks, libraries
|
||||
|
||||
|
||||
def entry_iterator():
|
||||
"""
|
||||
|
||||
@ -63,235 +52,6 @@ def canonical_entry_name(name):
|
||||
return name
|
||||
|
||||
|
||||
def parse_entry(content):
|
||||
"""
|
||||
Returns a dictionary of the features of the content.
|
||||
|
||||
Raises errors when a major error in the structure is expected, prints a warning for minor errors.
|
||||
"""
|
||||
|
||||
info = {}
|
||||
|
||||
# read name
|
||||
regex = re.compile(r"^# (.*)") # start of content, starting with "# " and then everything until the end of line
|
||||
matches = regex.findall(content)
|
||||
if len(matches) != 1 or not matches[0]: # name must be there
|
||||
raise RuntimeError('Name not found in entry "{}" : {}'.format(content, matches))
|
||||
info['name'] = matches[0]
|
||||
|
||||
# read description
|
||||
regex = re.compile(r"^.*\n\n_(.*)_\n") # third line from top, everything between underscores
|
||||
matches = regex.findall(content)
|
||||
if len(matches) != 1 or not matches[0]: # description must be there
|
||||
raise RuntimeError('Description not found in entry "{}"'.format(content))
|
||||
info['description'] = matches[0]
|
||||
|
||||
# first read all field names
|
||||
regex = re.compile(r"^- (.*?): ",
|
||||
re.MULTILINE) # start of each line having "- ", then everything until a colon, then ": "
|
||||
fields = regex.findall(content)
|
||||
|
||||
# check that essential fields are there
|
||||
for field in essential_fields:
|
||||
if field not in fields: # essential fields must be there
|
||||
raise RuntimeError('Essential field "{}" missing in entry "{}"'.format(field, info['name']))
|
||||
|
||||
# check that all fields are valid fields and are existing in that order
|
||||
index = 0
|
||||
for field in fields:
|
||||
while index < len(valid_fields) and field != valid_fields[index]:
|
||||
index += 1
|
||||
if index == len(valid_fields): # must be valid fields and must be in the right order
|
||||
raise RuntimeError(
|
||||
'Field "{}" in entry "{}" either misspelled or in wrong order'.format(field, info['name']))
|
||||
|
||||
# iterate over found fields
|
||||
for field in fields:
|
||||
regex = re.compile(r"- {}: (.*)".format(field))
|
||||
matches = regex.findall(content)
|
||||
if len(matches) != 1: # every field must be present only once
|
||||
raise RuntimeError('Field "{}" in entry "{}" exist multiple times.'.format(field, info['name']))
|
||||
v = matches[0]
|
||||
|
||||
# first store as is
|
||||
info[field.lower() + '-raw'] = v
|
||||
|
||||
# remove parenthesis with content
|
||||
v = re.sub(r'\([^)]*\)', '', v)
|
||||
|
||||
# split on ', '
|
||||
v = v.split(', ')
|
||||
|
||||
# strip
|
||||
v = [x.strip() for x in v]
|
||||
|
||||
# remove all being false (empty) that were for example just comments
|
||||
v = [x for x in v if x]
|
||||
|
||||
# if entry is of structure <..> remove <>
|
||||
v = [x[1:-1] if x[0] == '<' and x[-1] == '>' else x for x in v]
|
||||
|
||||
# empty fields will not be stored
|
||||
if not v:
|
||||
continue
|
||||
|
||||
# store in info
|
||||
info[field.lower()] = v
|
||||
|
||||
# check again that essential fields made it through
|
||||
for field in ('home', 'state', 'keywords', 'code language', 'code license'):
|
||||
if field not in info: # essential fields must still be inside
|
||||
raise RuntimeError('Essential field "{}" empty in entry "{}"'.format(field, info['name']))
|
||||
|
||||
# now checks on the content of fields
|
||||
|
||||
# name and description should not have spaces at the begin or end
|
||||
for field in ('name', 'description'):
|
||||
v = info[field]
|
||||
if len(v) != len(v.strip()): # warning about that
|
||||
print('Warning: No leading or trailing spaces in field {} in entry "{}"'.format(field, info['name']))
|
||||
|
||||
# state (essential field) must contain either beta or mature but not both, but at least one
|
||||
v = info['state']
|
||||
for t in v:
|
||||
if t != 'beta' and t != 'mature' and not t.startswith('inactive since '):
|
||||
raise RuntimeError('Unknown state tage "{}" in entry "{}"'.format(t, info['name']))
|
||||
if 'beta' in v != 'mature' in v:
|
||||
raise RuntimeError('State must be one of <"beta", "mature"> in entry "{}"'.format(info['name']))
|
||||
|
||||
# extract inactive year
|
||||
phrase = 'inactive since '
|
||||
inactive_year = [x[len(phrase):] for x in v if x.startswith(phrase)]
|
||||
assert len(inactive_year) <= 1
|
||||
if inactive_year:
|
||||
info['inactive'] = inactive_year[0]
|
||||
|
||||
# urls in home, download, play and code repositories must start with http or https (or git) and should not contain spaces
|
||||
for field in ['home', 'download', 'play', 'code repository']:
|
||||
if field in info:
|
||||
for url in info[field]:
|
||||
if not any(
|
||||
[url.startswith(x) for x in ['http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://']]):
|
||||
raise RuntimeError(
|
||||
'URL "{}" in entry "{}" does not start with http/https/git/svn/ftp/bzr'.format(url,
|
||||
info['name']))
|
||||
if ' ' in url:
|
||||
raise RuntimeError('URL "{}" in entry "{}" contains a space'.format(url, info['name']))
|
||||
|
||||
# github/gitlab repositories should end on .git and should start with https
|
||||
if 'code repository' in info:
|
||||
for repo in info['code repository']:
|
||||
if any((x in repo for x in ('github', 'gitlab', 'git.tuxfamily', 'git.savannah'))):
|
||||
if not repo.startswith('https://'):
|
||||
print('Warning: Repo {} in entry "{}" should start with https://'.format(repo, info['name']))
|
||||
if not repo.endswith('.git'):
|
||||
print('Warning: Repo {} in entry "{}" should end on .git.'.format(repo, info['name']))
|
||||
|
||||
# check that all platform tags are valid tags and are existing in that order
|
||||
if 'platform' in info:
|
||||
index = 0
|
||||
for platform in info['platform']:
|
||||
while index < len(valid_platforms) and platform != valid_platforms[index]:
|
||||
index += 1
|
||||
if index == len(valid_platforms): # must be valid platforms and must be in that order
|
||||
raise RuntimeError(
|
||||
'Platform tag "{}" in entry "{}" either misspelled or in wrong order'.format(platform,
|
||||
info['name']))
|
||||
|
||||
# there must be at least one keyword
|
||||
if 'keywords' not in info:
|
||||
raise RuntimeError('Need at least one keyword in entry "{}"'.format(info['name']))
|
||||
|
||||
# check for existence of at least one recommended keywords
|
||||
fail = True
|
||||
for recommended_keyword in recommended_keywords:
|
||||
if recommended_keyword in info['keywords']:
|
||||
fail = False
|
||||
break
|
||||
if fail: # must be at least one recommended keyword
|
||||
raise RuntimeError('Entry "{}" contains no recommended keyword'.format(info['name']))
|
||||
|
||||
# languages should be known
|
||||
languages = info['code language']
|
||||
for language in languages:
|
||||
if language not in known_languages:
|
||||
print('Warning: Language {} in entry "{}" is not a known language. Misspelled or new?'.format(language,
|
||||
info['name']))
|
||||
|
||||
# licenses should be known
|
||||
licenses = info['code license']
|
||||
for license in licenses:
|
||||
if license not in known_licenses:
|
||||
print('Warning: License {} in entry "{}" is not a known license. Misspelled or new?'.format(license,
|
||||
info['name']))
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def assemble_infos():
|
||||
"""
|
||||
Parses all entries and assembles interesting infos about them.
|
||||
"""
|
||||
|
||||
print('assemble game infos')
|
||||
|
||||
# a database of all important infos about the entries
|
||||
infos = []
|
||||
|
||||
# iterate over all entries
|
||||
for entry, _, content in entry_iterator():
|
||||
|
||||
# parse entry
|
||||
info = parse_entry(content)
|
||||
|
||||
# add file information
|
||||
info['file'] = entry
|
||||
|
||||
# check canonical file name
|
||||
canonical_file_name = canonical_entry_name(info['name']) + '.md'
|
||||
# we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
|
||||
if canonical_file_name != entry and canonical_file_name != entry[:-5] + '.md':
|
||||
print('Warning: file {} should be {}'.format(entry, canonical_file_name))
|
||||
source_file = os.path.join(entries_path, entry)
|
||||
target_file = os.path.join(entries_path, canonical_file_name)
|
||||
if not os.path.isfile(target_file):
|
||||
pass
|
||||
# os.rename(source_file, target_file)
|
||||
|
||||
# add to list
|
||||
infos.append(info)
|
||||
|
||||
return infos
|
||||
|
||||
|
||||
def extract_links():
|
||||
"""
|
||||
Parses all entries and extracts http(s) links from them
|
||||
"""
|
||||
|
||||
# regex for finding urls (can be in <> or in ]() or after a whitespace
|
||||
regex = re.compile(r"[\s\n]<(http.+?)>|]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n,]")
|
||||
|
||||
# iterate over all entries
|
||||
urls = set()
|
||||
for _, _, content in entry_iterator():
|
||||
|
||||
# apply regex
|
||||
matches = regex.findall(content)
|
||||
|
||||
# for each match
|
||||
for match in matches:
|
||||
|
||||
# for each possible clause
|
||||
for url in match:
|
||||
|
||||
# if there was something (and not a sourceforge git url)
|
||||
if url:
|
||||
urls.add(url)
|
||||
urls = sorted(list(urls), key=str.casefold)
|
||||
return urls
|
||||
|
||||
|
||||
def read_developers():
|
||||
"""
|
||||
|
||||
@ -564,11 +324,53 @@ def check_and_process_entry(entry):
|
||||
if not any(value.startswith(x) for x in valid_url_prefixes):
|
||||
message += 'URL "{}" in field "{}" does not start with a valid prefix'.format(value, field)
|
||||
|
||||
# github/gitlab repositories should end on .git and should start with https
|
||||
for repo in entry['Code repository']:
|
||||
if any(repo.startswith(x) for x in ('@', '?')):
|
||||
continue
|
||||
repo = repo.value.split(' ')[0].strip()
|
||||
if any((x in repo for x in ('github', 'gitlab', 'git.tuxfamily', 'git.savannah'))):
|
||||
if not repo.startswith('https://'):
|
||||
message += 'Repo "{}" should start with https://'.format(repo)
|
||||
if not repo.endswith('.git'):
|
||||
message += 'Repo "{}" should end on .git.'.format(repo)
|
||||
|
||||
# check that all platform tags are valid tags and are existing in that order
|
||||
if 'Platform' in entry:
|
||||
index = 0
|
||||
for platform in entry['Platform']:
|
||||
while index < len(valid_platforms) and platform != valid_platforms[index]:
|
||||
index += 1
|
||||
if index == len(valid_platforms): # must be valid platforms and must be in that order
|
||||
message += 'Platform tag "{}" either misspelled or in wrong order'.format(platform)
|
||||
|
||||
# there must be at least one keyword
|
||||
if not entry['Keywords']:
|
||||
message += 'Need at least one keyword'
|
||||
|
||||
# check for existence of at least one recommended keywords
|
||||
keywords = entry['Keywords']
|
||||
if not any(keyword in keywords for keyword in recommended_keywords):
|
||||
message += 'Entry contains no recommended keywords'
|
||||
|
||||
# languages should be known
|
||||
languages = entry['Code language']
|
||||
for language in languages:
|
||||
if language not in known_languages:
|
||||
message += 'Language "{}" is not a known code language. Misspelled or new?'.format(language)
|
||||
|
||||
# licenses should be known
|
||||
licenses = entry['Code license']
|
||||
for license in licenses:
|
||||
if license not in known_licenses:
|
||||
message += 'License "{}" is not a known license. Misspelled or new?'.format(license)
|
||||
|
||||
if message:
|
||||
raise RuntimeError(message)
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def extract_inactive_year(entry):
|
||||
state = entry['State']
|
||||
phrase = 'inactive since '
|
||||
@ -590,6 +392,7 @@ def write_entries(entries):
|
||||
for entry in entries:
|
||||
write_entry(entry)
|
||||
|
||||
|
||||
def write_entry(entry):
|
||||
"""
|
||||
|
||||
@ -618,6 +421,18 @@ def create_entry_content(entry):
|
||||
# title
|
||||
content = '# {}\n\n'.format(entry['Title'])
|
||||
|
||||
# we automatically sort some fields
|
||||
sort_fun = lambda x: str.casefold(x.value)
|
||||
for field in ('Media', 'Inspirations', 'Code Language'):
|
||||
if field in entry:
|
||||
values = entry[field]
|
||||
entry[field] = sorted(values, key=sort_fun)
|
||||
# we also sort keywords, but first the recommend ones and then other ones
|
||||
keywords = entry['Keywords']
|
||||
a = [x for x in keywords if x in recommended_keywords]
|
||||
b = [x for x in keywords if x not in recommended_keywords]
|
||||
entry['Keywords'] = sorted(a, key=sort_fun) + sorted(b, key=sort_fun)
|
||||
|
||||
# now properties in the recommended order
|
||||
for field in valid_properties:
|
||||
if field in entry:
|
||||
@ -651,4 +466,4 @@ def create_entry_content(entry):
|
||||
content += '\n'
|
||||
content += entry['Building']['Note']
|
||||
|
||||
return content
|
||||
return content
|
||||
|
||||
Reference in New Issue
Block a user