AnotherFoxGuy/opensourcegames
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

entries with values with comments

Browse Source
This commit is contained in:
Trilarion
2020-09-07 15:48:14 +02:00
parent 6b2edf8f56
commit 8a08425e09
17 changed files with 155 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
code/utils/constants.py
View File

@ -33,11 +33,11 @@ def get_config(key):
generic_comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
# these fields have to be present in each entry (in this order)
essential_fields = ('Title', 'Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
essential_fields = ('File', 'Title', 'Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
# only these fields can be used currently (in this order)
valid_fields = (
'Title', 'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
'File', 'Title', 'Home', 'Media', 'Inspirations', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Note', 'Building')
valid_building_fields = ('Build system', 'Build instructions')
@ -93,7 +93,9 @@ general_code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
# developer information (in the file all fields will be capitalized)
essential_developer_fields = ('Name', 'Games')
valid_developer_fields = essential_developer_fields + ('Contact', 'Home', 'Organization')
url_developer_fields = ('Home',)
# inspiration/original game information (in the file all fields will be capitalized)
essential_inspiration_fields = ('Name', 'Inspired entries')
valid_inspiration_fields = essential_inspiration_fields + ('Media',)
valid_inspiration_fields = essential_inspiration_fields + ('Media',)
url_inspiration_fields = ('Media',)

136
code/utils/osg.py
View File

@ -300,28 +300,32 @@ def read_developers():
grammar_file = os.path.join(code_path, 'grammar_listing.lark')
developers = osg_parse.read_and_parse(developer_file, grammar_file, osg_parse.ListingTransformer)
# now transform a bit more
for index, dev in enumerate(developers):
# check for valid keys
for field in dev.keys():
if field not in valid_developer_fields:
raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['Name']))
# strip from name or organization (just in case)
for field in ('Name', ):
if field in dev:
dev[field] = dev[field].strip()
# split games, contact (are lists)
for field in ('Games', 'Contact'):
if field in dev:
content = dev[field]
content = [x.strip() for x in content]
dev[field] = content
# now developers is a list of dictionaries for every entry with some properties
# check for duplicate names entries
names = [dev['Name'] for dev in developers]
duplicate_names = (name for name in names if names.count(name) > 1)
duplicate_names = set(duplicate_names) # to avoid duplicates in duplicate_names
if duplicate_names:
print('Warning: duplicate developer names: {}'.format(', '.join(duplicate_names)))
# check for essential, valid fields
for dev in developers:
# check that essential fields are existing
for field in essential_developer_fields:
if field not in dev:
raise RuntimeError('Essential field "{}" missing in developer {}'.format(field, dev['Name']))
# check that all fields are valid fields
for field in dev.keys():
if field not in valid_developer_fields:
raise RuntimeError('Invalid field "{}" in developer {}.'.format(field, dev['Name']))
# url fields
for field in url_developer_fields:
if field in dev:
content = dev[field]
if any(not (x.startswith('http://') or x.startswith('https://')) for x in content):
raise RuntimeError('Invalid URL in field "{}" in developer {}.'.format(field, dev['Name']))
return developers
@ -343,7 +347,7 @@ def write_developers(developers):
for dev in developers:
keys = list(dev.keys())
# developer name
content += '## {} [{}]\n\n'.format(dev['Name'], len(dev['games']))
content += '## {} [{}]\n\n'.format(dev['Name'], len(dev['Games']))
keys.remove('Name')
# all the remaining in alphabetical order, but 'games' first
@ -352,7 +356,6 @@ def write_developers(developers):
keys = ['Games'] + keys
for field in keys:
value = dev[field]
field = field.capitalize()
# lists get special treatment
if isinstance(value, list):
value.sort(key=str.casefold)
@ -377,20 +380,7 @@ def read_inspirations():
grammar_file = os.path.join(code_path, 'grammar_listing.lark')
inspirations = osg_parse.read_and_parse(inspirations_file, grammar_file, osg_parse.ListingTransformer)
# now inspirations is a list of dictionaries for every entry with keys (valid_developers_fields)
# now transform a bit more
for inspiration in inspirations:
# check that keys are valid keys
for field in inspiration.keys():
if field not in valid_inspiration_fields:
raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['Name']))
# split lists
for field in ('Inspired entries',):
if field in inspiration:
content = inspiration[field]
content = [x.strip() for x in content]
inspiration[field] = content
# now inspirations is a list of dictionaries for every entry with some properties
# check for duplicate names entries
names = [inspiration['Name'] for inspiration in inspirations]
@ -399,6 +389,23 @@ def read_inspirations():
if duplicate_names:
raise RuntimeError('Duplicate inspiration names: {}'.format(', '.join(duplicate_names)))
# check for essential, valid fields
for inspiration in inspirations:
# check that essential fields are existing
for field in essential_inspiration_fields:
if field not in inspiration:
raise RuntimeError('Essential field "{}" missing in inspiration {}'.format(field, inspiration['Name']))
# check that all fields are valid fields
for field in inspiration.keys():
if field not in valid_inspiration_fields:
raise RuntimeError('Invalid field "{}" in inspiration {}.'.format(field, inspiration['Name']))
# url fields
for field in url_inspiration_fields:
if field in inspiration:
content = inspiration[field]
if any(not (x.startswith('http://') or x.startswith('https://')) for x in content):
raise RuntimeError('Invalid URL in field "{}" in inspiration {}.'.format(field, inspiration['Name']))
# convert to dictionary
inspirations = {x['Name']: x for x in inspirations}
@ -417,7 +424,7 @@ def write_inspirations(inspirations):
# comment
content = '{}\n'.format(generic_comment_string)
# number of developer
# updated number of inspirations
content += '# Inspirations [{}]\n\n'.format(len(inspirations))
# sort by name
@ -436,7 +443,6 @@ def write_inspirations(inspirations):
keys = ['Inspired entries'] + keys
for field in keys:
value = inspiration[field]
field = field.capitalize()
# lists get special treatment
if isinstance(value, list):
value.sort(key=str.casefold) # sorted alphabetically
@ -472,15 +478,12 @@ def read_entries():
# parse and transform entry content
try:
entry = parse(content)
# add file information
entry['File'] = file
check_entry(entry)
post_process(entry)
entry = [('File', file),] + entry # add file information to the beginning
entry = check_and_process_entry(entry)
except Exception as e:
print('{} - {}'.format(file, e))
exception_happened = True
# raise RuntimeError(e)
continue
# add to list
@ -490,47 +493,44 @@ def read_entries():
return entries
def post_process(entry):
"""
:param entry:
:return:
"""
# remove all parentheses from developers
if 'Developer' in entry:
devs = entry['Developer']
devs = [re.sub(r'\([^)]*\)', '', x).strip() for x in devs]
if any(not x for x in devs):
raise RuntimeError('Empty developer')
entry['Developer'] = devs
def check_entry(entry):
"""
:param entry:
:return:
"""
def check_and_process_entry(entry):
message = ''
file = entry['File']
# check that all fields are valid fields and are existing in that order
index = 0
for e in entry:
field = e[0]
while index < len(valid_fields) and field != valid_fields[index]:
index += 1
if index == len(valid_fields): # must be valid fields and must be in the right order
message += 'Field "{}" either misspelled or in wrong order\n'.format(field)
# check canonical file name
canonical_file_name = canonical_entry_name(entry['Title']) + '.md'
# we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
message += 'file name should be {}\n'.format(canonical_file_name)
# order is fine we can convert to dictionary
d = {}
for field, value in entry:
if field in d:
message += 'Field "{}" appears twice\n'.format(field)
d[field] = value
entry = d
# check for essential fields
for field in essential_fields:
if field not in entry:
message += 'essential property "{}" missing\n'.format(field)
# check canonical file name
file = entry['File']
canonical_file_name = canonical_entry_name(entry['Title']) + '.md'
# we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
if canonical_file_name != file and canonical_file_name != file[:-5] + '.md':
message += 'file name should be {}\n'.format(canonical_file_name)
if message:
raise RuntimeError(message)
return entry
def write_entries(entries):
"""

64
code/utils/osg_parse.py
View File

@ -17,7 +17,7 @@ class ListingTransformer(lark.Transformer):
return x[0].value
def quoted_value(self, x):
return x[0].value[1:-1] # remove quotation marks
return x[0].value[1:-1].strip() # remove quotation marks and strip whitespaces
def property(self, x):
"""
@ -61,13 +61,23 @@ class EntryTransformer(lark.Transformer):
def quoted_value(self, x):
return x[0].value[1:-1] # remove quotation marks
def comment_value(self, x):
return x[0].value[1:-1] # remove parenthesis
def value(self, x):
if len(x) == 1:
v = ValueWithComment(value=x[0])
else:
v = ValueWithComment(value=x[0], comment=x[1])
return v
def property(self, x):
"""
The key of a property will be converted to lower case and the value part is the second part
:param x:
:return:
"""
return x[0], x[1:]
return x[0].value, x[1:]
def title(self, x):
return 'Title', x[0].value
@ -83,33 +93,33 @@ class EntryTransformer(lark.Transformer):
return 'Note', ''.join((x.value for x in x))
def building(self, x):
d = {}
for key, value in x:
if key in d:
raise RuntimeError('Key in entry appears twice')
d[key] = value
return 'Building', d
return 'Building', x
def start(self, x):
# we do the essential fields and valid fields checks right here
fields = [x[0] for x in x]
# check for essential fields
for field in c.essential_fields:
if field not in fields:
raise RuntimeError('Essential field "{}" is missing'.format(field))
# check for valid fields (in that order)
index = 0
for field in fields:
while index < len(c.valid_fields) and field != c.valid_fields[index]:
index += 1
if index == len(c.valid_fields):
raise RuntimeError('Field "{}" either not valid or in wrong order'.format(field))
d = {}
for key, value in x:
if key in d:
raise RuntimeError('Key in entry appears twice')
d[key] = value
return d
return x
class ValueWithComment:
"""
All our property values can have (optional) comments. This is the class that represents them to us and implements
equality and 'in' operator functionality purely on the value.
"""
def __init__(self, value, comment=None):
self.value = value
self.comment = comment
def __contains__(self, item):
return item in self.value
def __eq__(self, other):
return self.value == other
def __repr__(self):
if self.comment:
return '{} ({})'.format(self.value, self.comment)
else:
return '{}'.format(self.value)
def parse(parser, transformer, content):