synchronization of developers in entries with developers list
This commit is contained in:
Trilarion
@ -119,7 +119,7 @@ def create_toc(title, file, entries):
|
||||
# assemble rows
|
||||
rows = []
|
||||
for entry in entries:
|
||||
rows.append('- **[{}]({})** ({})'.format(entry['name'], '../' + entry['file'], ', '.join(
|
||||
rows.append('- **[{}]({})** ({})'.format(entry['Name'], '../' + entry['file'], ', '.join(
|
||||
entry['code language'] + entry['code license'] + entry['state'])))
|
||||
|
||||
# sort rows (by title)
|
||||
@ -407,7 +407,7 @@ def update_statistics(infos):
|
||||
rel(number_inactive))
|
||||
|
||||
if number_inactive > 0:
|
||||
entries_inactive = [(x['name'], x['inactive']) for x in infos if 'inactive' in x]
|
||||
entries_inactive = [(x['Name'], x['inactive']) for x in infos if 'inactive' in x]
|
||||
entries_inactive.sort(key=lambda x: str.casefold(x[0])) # first sort by name
|
||||
entries_inactive.sort(key=lambda x: x[1], reverse=True) # then sort by inactive year (more recently first)
|
||||
entries_inactive = ['{} ({})'.format(*x) for x in entries_inactive]
|
||||
@ -422,7 +422,7 @@ def update_statistics(infos):
|
||||
# number_no_language = sum(1 for x in infois if field not in x)
|
||||
# if number_no_language > 0:
|
||||
# statistics += 'Without language tag: {} ({:.1f}%)\n\n'.format(number_no_language, rel(number_no_language))
|
||||
# entries_no_language = [x['name'] for x in infois if field not in x]
|
||||
# entries_no_language = [x['Name'] for x in infois if field not in x]
|
||||
# entries_no_language.sort()
|
||||
# statistics += ', '.join(entries_no_language) + '\n\n'
|
||||
|
||||
@ -447,7 +447,7 @@ def update_statistics(infos):
|
||||
number_no_license = sum(1 for x in infos if field not in x)
|
||||
if number_no_license > 0:
|
||||
statistics += 'Without license tag: {} ({:.1f}%)\n\n'.format(number_no_license, rel(number_no_license))
|
||||
entries_no_license = [x['name'] for x in infos if field not in x]
|
||||
entries_no_license = [x['Name'] for x in infos if field not in x]
|
||||
entries_no_license.sort()
|
||||
statistics += ', '.join(entries_no_license) + '\n\n'
|
||||
|
||||
@ -491,7 +491,7 @@ def update_statistics(infos):
|
||||
entries = []
|
||||
for info in infos:
|
||||
if 'download' not in info and 'play' not in info:
|
||||
entries.append(info['name'])
|
||||
entries.append(info['Name'])
|
||||
entries.sort(key=str.casefold)
|
||||
statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
|
||||
|
||||
@ -511,7 +511,7 @@ def update_statistics(infos):
|
||||
break
|
||||
# if there were repositories, but none popular, add them to the list
|
||||
if not popular:
|
||||
entries.append(info['name'])
|
||||
entries.append(info['Name'])
|
||||
# print(info[field])
|
||||
entries.sort(key=str.casefold)
|
||||
statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
|
||||
@ -562,7 +562,7 @@ def update_statistics(infos):
|
||||
c_cpp_project_without_build_system = []
|
||||
for info in infos:
|
||||
if field not in info and ('C' in info['code language'] or 'C++' in info['code language']):
|
||||
c_cpp_project_without_build_system.append(info['name'])
|
||||
c_cpp_project_without_build_system.append(info['Name'])
|
||||
c_cpp_project_without_build_system.sort(key=str.casefold)
|
||||
statistics += '##### C and C++ projects without build system information ({})\n\n'.format(
|
||||
len(c_cpp_project_without_build_system)) + ', '.join(c_cpp_project_without_build_system) + '\n\n'
|
||||
@ -572,7 +572,7 @@ def update_statistics(infos):
|
||||
for info in infos:
|
||||
if field in info and 'CMake' in info[field] and (
|
||||
'C' in info['code language'] or 'C++' in info['code language']):
|
||||
c_cpp_project_not_cmake.append(info['name'])
|
||||
c_cpp_project_not_cmake.append(info['Name'])
|
||||
c_cpp_project_not_cmake.sort(key=str.casefold)
|
||||
statistics += '##### C and C++ projects with a build system different from CMake ({})\n\n'.format(
|
||||
len(c_cpp_project_not_cmake)) + ', '.join(c_cpp_project_not_cmake) + '\n\n'
|
||||
@ -615,7 +615,7 @@ def export_json(infos):
|
||||
for info in infos:
|
||||
|
||||
# game & description
|
||||
entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['name'], info['home'][0],
|
||||
entry = ['{} (<a href="{}">home</a>, <a href="{}">entry</a>)'.format(info['Name'], info['home'][0],
|
||||
r'https://github.com/Trilarion/opensourcegames/blob/master/entries/' +
|
||||
info['file']),
|
||||
textwrap.shorten(info['description'], width=60, placeholder='..')]
|
||||
@ -773,10 +773,10 @@ def export_primary_code_repositories_json(infos):
|
||||
continue
|
||||
|
||||
if not consumed:
|
||||
unconsumed_entries.append([info['name'], info[field]])
|
||||
unconsumed_entries.append([info['Name'], info[field]])
|
||||
# print output
|
||||
if 'code repository' in info:
|
||||
print('Entry "{}" unconsumed repo: {}'.format(info['name'], info[field]))
|
||||
print('Entry "{}" unconsumed repo: {}'.format(info['Name'], info[field]))
|
||||
|
||||
# sort them alphabetically (and remove duplicates)
|
||||
for k, v in primary_repos.items():
|
||||
@ -889,7 +889,7 @@ def update_inspirations(infos):
|
||||
# collect information
|
||||
originals = {}
|
||||
for info in infos:
|
||||
name = info['name']
|
||||
name = info['Name']
|
||||
keywords = info['keywords']
|
||||
ins = [x[12:] for x in keywords if x.startswith('inspired by ')]
|
||||
if ins:
|
||||
@ -924,7 +924,7 @@ def update_developer(infos):
|
||||
developer = {}
|
||||
for info in infos:
|
||||
if 'developer' in info:
|
||||
name = info['name']
|
||||
name = info['Name']
|
||||
devs = info['developer']
|
||||
for dev in devs:
|
||||
if dev in developer:
|
||||
@ -954,7 +954,7 @@ def check_code_dependencies(infos):
|
||||
valid_dependencies = list(utils.constants.general_code_dependencies_without_entry.keys())
|
||||
for info in infos:
|
||||
if any((x in ('framework', 'library', 'game engine') for x in info['keywords'])):
|
||||
name = info['name']
|
||||
name = info['Name']
|
||||
if name in utils.constants.code_dependencies_aliases:
|
||||
valid_dependencies.extend(utils.constants.code_dependencies_aliases[name])
|
||||
else:
|
||||
|
||||
@ -6,13 +6,14 @@ stored Git repositories.
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from utils import osg, osg_ui
|
||||
from bs4 import BeautifulSoup
|
||||
from utils import constants as c, utils, osg, osg_github
|
||||
|
||||
|
||||
def developer_info_lookup(name):
|
||||
for dev in developer_info:
|
||||
if name == dev['name']:
|
||||
if name == dev['Name']:
|
||||
return dev
|
||||
return None
|
||||
|
||||
@ -21,22 +22,7 @@ def developer_info_lookup(name):
|
||||
SF_alias_list = {'Erik Johansson (aka feneur)': 'Erik Johansson', 'Itms': 'Nicolas Auvray',
|
||||
'Wraitii': 'Lancelot de Ferrière', 'Simzer': 'Simon Laszlo', 'armin bajramovic': 'Armin Bajramovic'}
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# read developer info
|
||||
developer_info = osg.read_developer_info()
|
||||
osg.write_developer_info(developer_info) # write again just to make it nice and as sanity check
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
||||
# assemble info
|
||||
entries = osg.assemble_infos()
|
||||
|
||||
# cross-check
|
||||
osg.compare_entries_developers(entries, developer_info)
|
||||
|
||||
def test():
|
||||
# loop over infos
|
||||
developers = ''
|
||||
try:
|
||||
@ -44,7 +30,7 @@ if __name__ == "__main__":
|
||||
# active = False
|
||||
for entry in entries:
|
||||
|
||||
# if entry['name'] == 'Aleph One':
|
||||
# if entry['Name'] == 'Aleph One':
|
||||
# active = True
|
||||
# if not active:
|
||||
# continue
|
||||
@ -55,7 +41,7 @@ if __name__ == "__main__":
|
||||
break
|
||||
|
||||
# print
|
||||
entry_name = '{} - {}'.format(entry['file'], entry['name'])
|
||||
entry_name = '{} - {}'.format(entry['file'], entry['Name'])
|
||||
print(entry_name)
|
||||
content = ''
|
||||
|
||||
@ -136,3 +122,122 @@ if __name__ == "__main__":
|
||||
finally:
|
||||
# store developer info
|
||||
utils.write_text(os.path.join(c.root_path, 'collected_developer_info.txt'), developers)
|
||||
|
||||
def compare_entries_developers(entries, developers):
|
||||
"""
|
||||
Cross checks the game entries lists and the developers lists.
|
||||
:param entries: List of game entries
|
||||
:param developers: List of developers
|
||||
"""
|
||||
|
||||
# from the entries create a dictionary with developer names
|
||||
devs1 = {}
|
||||
for entry in entries:
|
||||
name = entry['Name']
|
||||
for dev in entry.get('developer', []):
|
||||
if dev in devs1:
|
||||
devs1[dev].append(name)
|
||||
else:
|
||||
devs1[dev] = [name]
|
||||
devs1_names = set(devs1.keys())
|
||||
|
||||
# from the developers create a dictionary with developer names
|
||||
devs2 = dict(zip((dev['Name'] for dev in developers), (dev['Games'] for dev in developers)))
|
||||
devs2_names = set(devs2.keys())
|
||||
|
||||
# devs only in entries
|
||||
for dev in devs1_names - devs2_names:
|
||||
print('Warning: dev "{}" only in entries ({}), not in developers'.format(dev, ','.join(devs1[dev])))
|
||||
# devs only in developers
|
||||
for dev in devs2_names - devs1_names:
|
||||
print('Warning: dev "{}" only in developers ({}), not in entries'.format(dev, ','.join(devs2[dev])))
|
||||
# for those in both, check that the games lists are equal
|
||||
for dev in devs1_names.intersection(devs2_names):
|
||||
games1 = set(devs1[dev])
|
||||
games2 = set(devs2[dev])
|
||||
delta = games1 - games2
|
||||
if delta:
|
||||
print('Warning: dev "{}" has games in entries ({}) that are not present in developers'.format(dev,
|
||||
', '.join(
|
||||
delta)))
|
||||
delta = games2 - games1
|
||||
if delta:
|
||||
print('Warning: dev "{}" has games in developers ({}) that are not present in entries'.format(dev, delta))
|
||||
|
||||
|
||||
class DevelopersMaintainer:
|
||||
|
||||
def __init__(self):
|
||||
self.developers = None
|
||||
self.entries = None
|
||||
|
||||
def read_developer(self):
|
||||
self.developers = osg.read_developers()
|
||||
print('{} developers read'.format(len(self.developers)))
|
||||
|
||||
def write_developer(self):
|
||||
if not self.developers:
|
||||
print('developers not yet loaded')
|
||||
return
|
||||
osg.write_developers(self.developers)
|
||||
print('developers written')
|
||||
|
||||
def check_for_duplicates(self):
|
||||
if not self.developers:
|
||||
print('developers not yet loaded')
|
||||
return
|
||||
developer_names = [x['Name'] for x in self.developers]
|
||||
for index, name in enumerate(developer_names):
|
||||
for other_name in developer_names[index + 1:]:
|
||||
if osg.name_similarity(name, other_name) > 0.8:
|
||||
print(' {} - {} is similar'.format(name, other_name))
|
||||
print('duplicates checked')
|
||||
|
||||
def check_for_orphans(self):
|
||||
if not self.developers:
|
||||
print('developers not yet loaded')
|
||||
return
|
||||
for dev in self.developers:
|
||||
if not dev['Games']:
|
||||
print(' {} has no "Games" field'.format(dev['Name']))
|
||||
print('orphanes checked')
|
||||
|
||||
def check_for_missing_developers_in_entries(self):
|
||||
if not self.developers:
|
||||
print('developer not yet loaded')
|
||||
return
|
||||
if not self.entries:
|
||||
print('entries not yet loaded')
|
||||
return
|
||||
for dev in self.developers:
|
||||
dev_name = dev['Name']
|
||||
for entry_name in dev['Games']:
|
||||
x = [x for x in self.entries if x['Title'] == entry_name]
|
||||
assert len(x) <= 1
|
||||
if not x:
|
||||
print('Entry "{}" listed as game of developer "{}" but this entry does not exist'.format(entry_name, dev_name))
|
||||
else:
|
||||
entry = x[0]
|
||||
if 'Developer' not in entry or dev_name not in entry['Developer']:
|
||||
print('Entry "{}" listed in developer "{}" but not listed in that entry'.format(entry_name, dev_name))
|
||||
print('missed developer checked')
|
||||
|
||||
def read_entries(self):
|
||||
self.entries = osg.read_entries()
|
||||
print('{} entries read'.format(len(self.entries)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
m = DevelopersMaintainer()
|
||||
|
||||
actions = {
|
||||
'Read developers': m.read_developer,
|
||||
'Write developers': m.write_developer,
|
||||
'Check for duplicates': m.check_for_duplicates,
|
||||
'Check for orphans': m.check_for_orphans,
|
||||
'Check for games in developers not listed': m.check_for_missing_developers_in_entries,
|
||||
'Read entries': m.read_entries
|
||||
}
|
||||
|
||||
osg_ui.run_simple_button_app('Maintenance developer', actions)
|
||||
@ -43,8 +43,8 @@ class InspirationMaintainer:
|
||||
print('inspirations not yet loaded')
|
||||
return
|
||||
for inspiration in self.inspirations.values():
|
||||
if not inspiration['inspired entries']:
|
||||
print(' {} has no inspired entries'.format(inspiration['name']))
|
||||
if not inspiration['Inspired entries']:
|
||||
print(' {} has no inspired entries'.format(inspiration['Name']))
|
||||
print('orphanes checked')
|
||||
|
||||
def check_for_missing_inspirations_in_entries(self):
|
||||
@ -55,18 +55,27 @@ class InspirationMaintainer:
|
||||
print('entries not yet loaded')
|
||||
return
|
||||
for inspiration in self.inspirations.values():
|
||||
inspiration_name = inspiration['name']
|
||||
for entry_name in inspiration['inspired entries']:
|
||||
x = [x for x in self.entries if x['title'] == entry_name]
|
||||
inspiration_name = inspiration['Name']
|
||||
for entry_name in inspiration['Inspired entries']:
|
||||
x = [x for x in self.entries if x['Title'] == entry_name]
|
||||
assert len(x) <= 1
|
||||
if not x:
|
||||
print('Entry "{}" listed in inspiration "{}" but this entry does not exist'.format(entry_name, inspiration_name))
|
||||
else:
|
||||
entry = x[0]
|
||||
if 'inspirations' not in entry or inspiration_name not in entry['inspirations']:
|
||||
if 'Inspirations' not in entry or inspiration_name not in entry['Inspirations']:
|
||||
print('Entry "{}" listed in inspiration "{}" but not listed in this entry'.format(entry_name, inspiration_name))
|
||||
print('missed inspirations checked')
|
||||
|
||||
def check_for_wikipedia_links(self):
|
||||
if not self.inspirations:
|
||||
print('inspirations not yet loaded')
|
||||
return
|
||||
for inspiration in self.inspirations.values():
|
||||
if 'Media' in inspiration and any(('https://en.wikipedia.org/wiki/' in x for x in inspiration['Media'])):
|
||||
continue
|
||||
# search in wikipedia
|
||||
|
||||
def update_inspired_entries(self):
|
||||
if not self.inspirations:
|
||||
print('inspirations not yet loaded')
|
||||
@ -76,15 +85,15 @@ class InspirationMaintainer:
|
||||
return
|
||||
# loop over all inspirations and delete inspired entries
|
||||
for inspiration in self.inspirations.values():
|
||||
inspiration['inspired entries'] = []
|
||||
inspiration['Inspired entries'] = []
|
||||
# loop over all entries and add to inspirations of entry
|
||||
for entry in self.entries:
|
||||
entry_name = entry['title']
|
||||
for inspiration in entry.get('inspirations', []):
|
||||
entry_name = entry['Title']
|
||||
for inspiration in entry.get('Inspirations', []):
|
||||
if inspiration in self.inspirations:
|
||||
self.inspirations[inspiration]['inspired entries'].append(entry_name)
|
||||
self.inspirations[inspiration]['Inspired entries'].append(entry_name)
|
||||
else:
|
||||
self.inspirations[inspiration] = {'name': inspiration, 'inspired entries': [entry_name]}
|
||||
self.inspirations[inspiration] = {'Name': inspiration, 'Inspired entries': [entry_name]}
|
||||
print('inspired entries updated')
|
||||
|
||||
def read_entries(self):
|
||||
@ -12,7 +12,7 @@ tocs_path = os.path.join(entries_path, 'tocs')
|
||||
code_path = os.path.join(root_path, 'code')
|
||||
|
||||
inspirations_file = os.path.join(root_path, 'inspirations.md')
|
||||
developer_file = os.path.join(root_path, 'developer.md')
|
||||
developer_file = os.path.join(root_path, 'developers.md')
|
||||
|
||||
# local config
|
||||
local_config_file = os.path.join(root_path, 'local-config.ini')
|
||||
@ -33,12 +33,12 @@ def get_config(key):
|
||||
generic_comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
|
||||
|
||||
# these fields have to be present in each entry (in this order)
|
||||
essential_fields = ('Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
|
||||
essential_fields = ('Title', 'Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
|
||||
|
||||
# only these fields can be used currently (in this order)
|
||||
valid_fields = (
|
||||
'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
|
||||
'Code license', 'Code dependencies', 'Assets license', 'Developer')
|
||||
'Title', 'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
|
||||
'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Note', 'Building')
|
||||
|
||||
valid_building_fields = ('Build system', 'Build instructions')
|
||||
|
||||
@ -91,7 +91,9 @@ general_code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
|
||||
'tkinter': 'https://docs.python.org/3/library/tk.html'}
|
||||
|
||||
# developer information (in the file all fields will be capitalized)
|
||||
valid_developer_fields = ('name', 'games', 'contact', 'organization', 'home')
|
||||
essential_developer_fields = ('Name', 'Games')
|
||||
valid_developer_fields = essential_developer_fields + ('Contact', 'Home', 'Organization')
|
||||
|
||||
# inspiration/original game information (in the file all fields will be capitalized)
|
||||
valid_inspiration_fields = ('name', 'inspired entries', 'media')
|
||||
essential_inspiration_fields = ('Name', 'Inspired entries')
|
||||
valid_inspiration_fields = essential_inspiration_fields + ('Media',)
|
||||
@ -4,104 +4,9 @@ Specific functions working on the games.
|
||||
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from utils import utils
|
||||
import lark
|
||||
|
||||
from utils import utils, osg_parse
|
||||
from utils.constants import *
|
||||
|
||||
|
||||
class ListingTransformer(lark.Transformer):
|
||||
"""
|
||||
Transforms content parsed by grammar_listing.lark further.
|
||||
Used for the developer and inspirations list.
|
||||
"""
|
||||
|
||||
def unquoted_value(self, x):
|
||||
return x[0].value
|
||||
|
||||
def quoted_value(self, x):
|
||||
return x[0].value[1:-1] # remove quotation marks
|
||||
|
||||
def property(self, x):
|
||||
"""
|
||||
The key of a property will be converted to lower case and the value part is the second part
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return x[0].lower(), x[1:]
|
||||
|
||||
def name(self, x):
|
||||
"""
|
||||
The name part is treated as a property with key "name"
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return 'name', x[0].value
|
||||
|
||||
def entry(self, x):
|
||||
"""
|
||||
All (key, value) tuples are inserted into a dictionary.
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
def start(self, x):
|
||||
return x
|
||||
|
||||
|
||||
# transformer
|
||||
class EntryTransformer(lark.Transformer):
|
||||
|
||||
def unquoted_value(self, x):
|
||||
return x[0].value
|
||||
|
||||
def quoted_value(self, x):
|
||||
return x[0].value[1:-1] # remove quotation marks
|
||||
|
||||
def property(self, x):
|
||||
"""
|
||||
The key of a property will be converted to lower case and the value part is the second part
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return x[0].lower(), x[1:]
|
||||
|
||||
def title(self, x):
|
||||
return 'title', x[0].value
|
||||
|
||||
def note(self, x):
|
||||
"""
|
||||
Optional
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
if not x:
|
||||
raise lark.Discard
|
||||
return 'note', ''.join((x.value for x in x))
|
||||
|
||||
def building(self, x):
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return 'building', d
|
||||
|
||||
def start(self, x):
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
|
||||
regex_sanitize_name = re.compile(r"[^A-Za-z 0-9-+]+")
|
||||
regex_sanitize_name_space_eater = re.compile(r" +")
|
||||
|
||||
@ -387,48 +292,32 @@ def extract_links():
|
||||
return urls
|
||||
|
||||
|
||||
def read_and_parse(content_file: str, grammar_file: str, transformer: lark.Transformer):
|
||||
"""
|
||||
Reads a content file and a grammar file and parses the content with the grammar following by
|
||||
transforming the parsed output and returning the transformed result.
|
||||
:param content_file:
|
||||
:param grammar_file:
|
||||
:param transformer:
|
||||
:return:
|
||||
"""
|
||||
content = utils.read_text(content_file)
|
||||
grammar = utils.read_text(grammar_file)
|
||||
parser = lark.Lark(grammar, debug=False, parser='lalr')
|
||||
tree = parser.parse(content)
|
||||
return transformer.transform(tree)
|
||||
|
||||
|
||||
def read_developer_info():
|
||||
def read_developers():
|
||||
"""
|
||||
|
||||
:return:
|
||||
"""
|
||||
grammar_file = os.path.join(code_path, 'grammar_listing.lark')
|
||||
transformer = ListingTransformer()
|
||||
developers = read_and_parse(developer_file, grammar_file, transformer)
|
||||
developers = osg_parse.read_and_parse(developer_file, grammar_file, osg_parse.ListingTransformer)
|
||||
|
||||
# now transform a bit more
|
||||
for index, dev in enumerate(developers):
|
||||
# check for valid keys
|
||||
for field in dev.keys():
|
||||
if field not in valid_developer_fields:
|
||||
raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['name']))
|
||||
raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['Name']))
|
||||
# strip from name or organization (just in case)
|
||||
for field in ('name', ):
|
||||
for field in ('Name', ):
|
||||
if field in dev:
|
||||
dev[field] = dev[field].strip()
|
||||
# split games, contact (are lists)
|
||||
for field in ('games', 'contact'):
|
||||
for field in ('Games', 'Contact'):
|
||||
if field in dev:
|
||||
content = dev[field]
|
||||
content = [x.strip() for x in content]
|
||||
dev[field] = content
|
||||
# check for duplicate names entries
|
||||
names = [dev['name'] for dev in developers]
|
||||
names = [dev['Name'] for dev in developers]
|
||||
duplicate_names = (name for name in names if names.count(name) > 1)
|
||||
duplicate_names = set(duplicate_names) # to avoid duplicates in duplicate_names
|
||||
if duplicate_names:
|
||||
@ -436,7 +325,7 @@ def read_developer_info():
|
||||
return developers
|
||||
|
||||
|
||||
def write_developer_info(developers):
|
||||
def write_developers(developers):
|
||||
"""
|
||||
|
||||
:return:
|
||||
@ -448,19 +337,19 @@ def write_developer_info(developers):
|
||||
content += '# Developer [{}]\n\n'.format(len(developers))
|
||||
|
||||
# sort by name
|
||||
developers.sort(key=lambda x: str.casefold(x['name']))
|
||||
developers.sort(key=lambda x: str.casefold(x['Name']))
|
||||
|
||||
# iterate over them
|
||||
for dev in developers:
|
||||
keys = list(dev.keys())
|
||||
# developer name
|
||||
content += '## {} [{}]\n\n'.format(dev['name'], len(dev['games']))
|
||||
keys.remove('name')
|
||||
content += '## {} [{}]\n\n'.format(dev['Name'], len(dev['games']))
|
||||
keys.remove('Name')
|
||||
|
||||
# all the remaining in alphabetical order, but 'games' first
|
||||
keys.remove('games')
|
||||
keys.remove('Games')
|
||||
keys.sort()
|
||||
keys = ['games'] + keys
|
||||
keys = ['Games'] + keys
|
||||
for field in keys:
|
||||
value = dev[field]
|
||||
field = field.capitalize()
|
||||
@ -484,34 +373,34 @@ def read_inspirations():
|
||||
"""
|
||||
# read inspirations
|
||||
|
||||
# read and parse inspirations
|
||||
grammar_file = os.path.join(code_path, 'grammar_listing.lark')
|
||||
transformer = ListingTransformer()
|
||||
inspirations = read_and_parse(inspirations_file, grammar_file, transformer)
|
||||
inspirations = osg_parse.read_and_parse(inspirations_file, grammar_file, osg_parse.ListingTransformer)
|
||||
|
||||
# now inspirations is a list of dictionaries for every entry with keys (valid_developers_fields)
|
||||
|
||||
# now transform a bit more
|
||||
for index, inspiration in enumerate(inspirations):
|
||||
for inspiration in inspirations:
|
||||
# check that keys are valid keys
|
||||
for field in inspiration.keys():
|
||||
if field not in valid_inspiration_fields:
|
||||
raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['name']))
|
||||
raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['Name']))
|
||||
# split lists
|
||||
for field in ('inspired entries',):
|
||||
for field in ('Inspired entries',):
|
||||
if field in inspiration:
|
||||
content = inspiration[field]
|
||||
content = [x.strip() for x in content]
|
||||
inspiration[field] = content
|
||||
|
||||
# check for duplicate names entries
|
||||
names = [inspiration['name'] for inspiration in inspirations]
|
||||
names = [inspiration['Name'] for inspiration in inspirations]
|
||||
duplicate_names = (name for name in names if names.count(name) > 1)
|
||||
duplicate_names = set(duplicate_names) # to avoid duplicates in duplicate_names
|
||||
if duplicate_names:
|
||||
raise RuntimeError('Duplicate inspiration names: {}'.format(', '.join(duplicate_names)))
|
||||
|
||||
# convert to dictionary
|
||||
inspirations = {x['name']: x for x in inspirations}
|
||||
inspirations = {x['Name']: x for x in inspirations}
|
||||
|
||||
return inspirations
|
||||
|
||||
@ -532,25 +421,25 @@ def write_inspirations(inspirations):
|
||||
content += '# Inspirations [{}]\n\n'.format(len(inspirations))
|
||||
|
||||
# sort by name
|
||||
inspirations.sort(key=lambda x: str.casefold(x['name']))
|
||||
inspirations.sort(key=lambda x: str.casefold(x['Name']))
|
||||
|
||||
# iterate over them
|
||||
for inspiration in inspirations:
|
||||
keys = list(inspiration.keys())
|
||||
# inspiration name
|
||||
content += '## {} [{}]\n\n'.format(inspiration['name'], len(inspiration['inspired entries']))
|
||||
keys.remove('name')
|
||||
content += '## {} [{}]\n\n'.format(inspiration['Name'], len(inspiration['Inspired entries']))
|
||||
keys.remove('Name')
|
||||
|
||||
# all the remaining in alphabetical order, but "inspired entries" first
|
||||
keys.remove('inspired entries')
|
||||
keys.remove('Inspired entries')
|
||||
keys.sort()
|
||||
keys = ['inspired entries'] + keys
|
||||
keys = ['Inspired entries'] + keys
|
||||
for field in keys:
|
||||
value = inspiration[field]
|
||||
field = field.capitalize()
|
||||
# lists get special treatment
|
||||
if isinstance(value, list):
|
||||
value.sort(key=str.casefold) # sorted alphabetically
|
||||
value.sort(key=str.casefold) # sorted alphabetically
|
||||
value = [x if not ',' in x else '"{}"'.format(x) for x in value] # surround those with a comma with quotation marks
|
||||
value = ', '.join(value)
|
||||
content += '- {}: {}\n'.format(field, value)
|
||||
@ -565,13 +454,10 @@ def read_entries():
|
||||
Parses all entries and assembles interesting infos about them.
|
||||
"""
|
||||
|
||||
# setup parser
|
||||
# setup parser and transformer
|
||||
grammar_file = os.path.join(code_path, 'grammar_entries.lark')
|
||||
grammar = utils.read_text(grammar_file)
|
||||
parser = lark.Lark(grammar, debug=False, parser='lalr')
|
||||
|
||||
# setup transformer
|
||||
transformer = EntryTransformer()
|
||||
parse = osg_parse.create(grammar, osg_parse.EntryTransformer)
|
||||
|
||||
# a database of all important infos about the entries
|
||||
entries = []
|
||||
@ -585,13 +471,13 @@ def read_entries():
|
||||
|
||||
# parse and transform entry content
|
||||
try:
|
||||
tree = parser.parse(content)
|
||||
|
||||
entry = transformer.transform(tree)
|
||||
entry = parse(content)
|
||||
# add file information
|
||||
entry['file'] = file
|
||||
entry['File'] = file
|
||||
|
||||
check_entry(entry)
|
||||
|
||||
post_process(entry)
|
||||
except Exception as e:
|
||||
print('{} - {}'.format(file, e))
|
||||
exception_happened = True
|
||||
@ -604,6 +490,22 @@ def read_entries():
|
||||
|
||||
return entries
|
||||
|
||||
def post_process(entry):
|
||||
"""
|
||||
|
||||
:param entry:
|
||||
:return:
|
||||
"""
|
||||
|
||||
# remove all parentheses from developers
|
||||
if 'Developer' in entry:
|
||||
devs = entry['Developer']
|
||||
devs = [re.sub(r'\([^)]*\)', '', x).strip() for x in devs]
|
||||
if any(not x for x in devs):
|
||||
raise RuntimeError('Empty developer')
|
||||
entry['Developer'] = devs
|
||||
|
||||
|
||||
|
||||
def check_entry(entry):
|
||||
"""
|
||||
@ -613,14 +515,19 @@ def check_entry(entry):
|
||||
"""
|
||||
message = ''
|
||||
|
||||
file = entry['file']
|
||||
file = entry['File']
|
||||
|
||||
# check canonical file name
|
||||
canonical_file_name = canonical_entry_name(entry['title']) + '.md'
|
||||
canonical_file_name = canonical_entry_name(entry['Title']) + '.md'
|
||||
# we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
|
||||
if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
|
||||
message += 'file name should be {}\n'.format(canonical_file_name)
|
||||
|
||||
# check for essential fields
|
||||
for field in essential_fields:
|
||||
if field not in entry:
|
||||
message += 'essential property "{}" missing\n'.format(field)
|
||||
|
||||
if message:
|
||||
raise RuntimeError(message)
|
||||
|
||||
@ -661,20 +568,19 @@ def create_entry_content(entry):
|
||||
"""
|
||||
|
||||
# title
|
||||
content = '# {}\n\n'.format(entry['title'])
|
||||
content = '# {}\n\n'.format(entry['Title'])
|
||||
|
||||
# now properties in the recommended order
|
||||
for field in valid_fields:
|
||||
field_name = field.lower()
|
||||
if field_name in entry:
|
||||
c = entry[field_name]
|
||||
if field in entry:
|
||||
c = entry[field]
|
||||
c = ['"{}"'.format(x) if ',' in x else x for x in c]
|
||||
content += '- {}: {}\n'.format(field, ', '.join(c))
|
||||
content += '\n'
|
||||
|
||||
# if there is a note, insert it
|
||||
if 'note' in entry:
|
||||
content += entry['note']
|
||||
if 'Note' in entry:
|
||||
content += entry['Note']
|
||||
|
||||
# building header
|
||||
content += '## Building\n'
|
||||
@ -682,59 +588,15 @@ def create_entry_content(entry):
|
||||
# building properties if present
|
||||
has_properties = False
|
||||
for field in valid_building_fields:
|
||||
field_name = field.lower()
|
||||
if field_name in entry['building']:
|
||||
if field in entry['Building']:
|
||||
if not has_properties:
|
||||
has_properties = True
|
||||
content += '\n'
|
||||
content += '- {}: {}\n'.format(field, ', '.join(entry['building'][field_name]))
|
||||
content += '- {}: {}\n'.format(field, ', '.join(entry['Building'][field]))
|
||||
|
||||
# if there is a note, insert it
|
||||
if 'note' in entry['building']:
|
||||
if 'Note' in entry['Building']:
|
||||
content += '\n'
|
||||
content += entry['building']['note']
|
||||
content += entry['Building']['Note']
|
||||
|
||||
return content
|
||||
|
||||
def compare_entries_developers(entries, developers):
|
||||
"""
|
||||
Cross checks the game entries lists and the developers lists.
|
||||
:param entries: List of game entries
|
||||
:param developers: List of developers
|
||||
"""
|
||||
|
||||
# from the entries create a dictionary with developer names
|
||||
devs1 = {}
|
||||
for entry in entries:
|
||||
name = entry['name']
|
||||
for dev in entry.get('developer', []):
|
||||
if dev in devs1:
|
||||
devs1[dev].append(name)
|
||||
else:
|
||||
devs1[dev] = [name]
|
||||
devs1_names = set(devs1.keys())
|
||||
|
||||
# from the developers create a dictionary with developer names
|
||||
devs2 = dict(zip((dev['name'] for dev in developers), (dev['games'] for dev in developers)))
|
||||
devs2_names = set(devs2.keys())
|
||||
|
||||
# devs only in entries
|
||||
for dev in devs1_names - devs2_names:
|
||||
print('Warning: dev "{}" only in entries ({}), not in developers'.format(dev, ','.join(devs1[dev])))
|
||||
# devs only in developers
|
||||
for dev in devs2_names - devs1_names:
|
||||
print('Warning: dev "{}" only in developers ({}), not in entries'.format(dev, ','.join(devs2[dev])))
|
||||
# for those in both, check that the games lists are equal
|
||||
for dev in devs1_names.intersection(devs2_names):
|
||||
games1 = set(devs1[dev])
|
||||
games2 = set(devs2[dev])
|
||||
delta = games1 - games2
|
||||
if delta:
|
||||
print('Warning: dev "{}" has games in entries ({}) that are not present in developers'.format(dev,
|
||||
', '.join(
|
||||
delta)))
|
||||
delta = games2 - games1
|
||||
if delta:
|
||||
print('Warning: dev "{}" has games in developers ({}) that are not present in entries'.format(dev,
|
||||
', '.join(
|
||||
delta)))
|
||||
return content
|
||||
@ -0,0 +1,140 @@
|
||||
"""
|
||||
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
import lark
|
||||
from utils import utils, constants as c
|
||||
|
||||
|
||||
class ListingTransformer(lark.Transformer):
|
||||
"""
|
||||
Transforms content parsed by grammar_listing.lark further.
|
||||
Used for the developer and inspirations list.
|
||||
"""
|
||||
|
||||
def unquoted_value(self, x):
|
||||
return x[0].value
|
||||
|
||||
def quoted_value(self, x):
|
||||
return x[0].value[1:-1] # remove quotation marks
|
||||
|
||||
def property(self, x):
|
||||
"""
|
||||
The key of a property will be converted to lower case and the value part is the second part
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return x[0], x[1:]
|
||||
|
||||
def name(self, x):
|
||||
"""
|
||||
The name part is treated as a property with key "Name"
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return 'Name', x[0].value
|
||||
|
||||
def entry(self, x):
|
||||
"""
|
||||
All (key, value) tuples are inserted into a dictionary.
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
def start(self, x):
|
||||
return x
|
||||
|
||||
|
||||
# transformer
|
||||
class EntryTransformer(lark.Transformer):
|
||||
|
||||
def unquoted_value(self, x):
|
||||
return x[0].value
|
||||
|
||||
def quoted_value(self, x):
|
||||
return x[0].value[1:-1] # remove quotation marks
|
||||
|
||||
def property(self, x):
|
||||
"""
|
||||
The key of a property will be converted to lower case and the value part is the second part
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
return x[0], x[1:]
|
||||
|
||||
def title(self, x):
|
||||
return 'Title', x[0].value
|
||||
|
||||
def note(self, x):
|
||||
"""
|
||||
Optional
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
if not x:
|
||||
raise lark.Discard
|
||||
return 'Note', ''.join((x.value for x in x))
|
||||
|
||||
def building(self, x):
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return 'Building', d
|
||||
|
||||
def start(self, x):
|
||||
# we do the essential fields and valid fields checks right here
|
||||
fields = [x[0] for x in x]
|
||||
# check for essential fields
|
||||
for field in c.essential_fields:
|
||||
if field not in fields:
|
||||
raise RuntimeError('Essential field "{}" is missing'.format(field))
|
||||
# check for valid fields (in that order)
|
||||
index = 0
|
||||
for field in fields:
|
||||
while index < len(c.valid_fields) and field != c.valid_fields[index]:
|
||||
index += 1
|
||||
if index == len(c.valid_fields):
|
||||
raise RuntimeError('Field "{}" either not valid or in wrong order'.format(field))
|
||||
d = {}
|
||||
for key, value in x:
|
||||
if key in d:
|
||||
raise RuntimeError('Key in entry appears twice')
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
|
||||
def parse(parser, transformer, content):
|
||||
tree = parser.parse(content)
|
||||
value = transformer.transform(tree)
|
||||
return value
|
||||
|
||||
|
||||
def create(grammar, Transformer):
|
||||
parser = lark.Lark(grammar, debug=False, parser='lalr')
|
||||
transformer = Transformer()
|
||||
return partial(parse, parser, transformer)
|
||||
|
||||
|
||||
def read_and_parse(content_file: str, grammar_file: str, Transformer: lark.Transformer):
|
||||
"""
|
||||
Reads a content file and a grammar file and parses the content with the grammar following by
|
||||
transforming the parsed output and returning the transformed result.
|
||||
:param content_file:
|
||||
:param grammar_file:
|
||||
:param transformer:
|
||||
:return:
|
||||
"""
|
||||
grammar = utils.read_text(grammar_file)
|
||||
parse = create(grammar, Transformer)
|
||||
|
||||
content = utils.read_text(content_file)
|
||||
return parse(content)
|
||||
16
code/utils/osg_wikipedia.py
Normal file
16
code/utils/osg_wikipedia.py
Normal file
@ -0,0 +1,16 @@
|
||||
"""
|
||||
Everything specific to search in Wikipedia.
|
||||
Using https://github.com/goldsmith/Wikipedia
|
||||
"""
|
||||
|
||||
import wikipedia
|
||||
|
||||
|
||||
def search(search_term, results=3):
|
||||
"""
|
||||
|
||||
:param search_term:
|
||||
:param max_results:
|
||||
:return:
|
||||
"""
|
||||
return wikipedia.search(search_term, results=results)
|
||||
Reference in New Issue
Block a user