From 881f0c77c5f6ee28046d2c735d00504ed33c4085 Mon Sep 17 00:00:00 2001
From: Trilarion <Trilarion@users.noreply.github.com>
Date: Mon, 31 Aug 2020 23:50:45 +0200
Subject: [PATCH] grammar for inspirations and developers lists

---
 code/backlog.txt                              |   7 +
 code/grammar_listing.lark                     |  29 +++--
 code/maintenance.py                           |  14 +-
 code/maintenance_collect_inspiration_infos.py |   2 +-
 code/utils/constants.py                       |  69 ++++++++++
 code/utils/osg.py                             | 122 ++++++++----------
 inspirations.md                               |   4 +-
 7 files changed, 154 insertions(+), 93 deletions(-)

diff --git a/code/backlog.txt b/code/backlog.txt
index 2d267751..8a9ff1dd 100644
--- a/code/backlog.txt
+++ b/code/backlog.txt
@@ -5,14 +5,21 @@ http://cdetect.sourceforge.net/
 http://circularstudios.com/
 http://cyxdown.free.fr/bs/
 http://cyxdown.free.fr/f2b/
+https://github.com/nfprojects/nfengine
 http://dead-code.org/home/
 http://e-adventure.e-ucm.es/login/index.php (games of eAdventure)
 http://ethernet.wasted.ch/
 http://evolonline.org/about
 http://game-editor.com/Main_Page
 http://giderosmobile.com/
+https://github.com/skylicht-lab/skylicht-engine
+https://github.com/etlegacy/etlegacy
+https://github.com/Soldat/soldat
+https://github.com/guillaumechereau/goxel
 http://haxepunk.com/
 http://hcsoftware.sourceforge.net/jason-rohrer/ (various games there)
+https://github.com/cxong/cdogs-sdl
+https://github.com/terrafx/terrafx
 http://hgm.nubati.net/
 http://icculus.org/
 http://icculus.org/asciiroth/
diff --git a/code/grammar_listing.lark b/code/grammar_listing.lark
index 2e0c873e..2e2cb643 100644
--- a/code/grammar_listing.lark
+++ b/code/grammar_listing.lark
@@ -1,17 +1,18 @@
-start: header entry*
+start: entry*
+entry: "##" name "(" _NUMBER ")\n" property+ 
+property: "-" _key ":" _values "\n"
+_key: /(?! ).+?(?=:)(?<! )/    // key: everything until next ":", not beginning or ending with a space
+_values: [_value ("," _value)*]
+_value: quoted_value | unquoted_value
+quoted_value: /\".+?\"/   // with quotation marks, can contain commas
+unquoted_value: /(?![ \"])[^,\n]+(?<![ \"])/ // cannot contain commas, cannot start or end with quotation mark
 
-header: "# " name " (" number ")\n" _E
+name: /(?! ).+?(?= \()/    // developer name: everything until " ("
 
-entry: "## " name " (" number  ")\n" _E property+ _E
+_NUMBER: /[0-9]+/
 
-property: "- " _key ": " _value "\n"
-_key: /(?! ).+?(?=:)(?<! )/                      // key: everything until next ":", not beginning or ending with a space
-_value: /.+(?<! )/                               // everything until the end of the line, not ending with a space
-
-name: /.+?(?= \()/                               // developer name: everything until " ("
-number: /[0-9]+/
-
-COMMENT: /^\[comment\]: #.*$\n/m                 // [comment]: # xxx
-_E: /^$\n/m                                      // empty new line
-
-%ignore COMMENT
\ No newline at end of file
+%import common.WS
+%ignore WS
+%ignore /^\[comment\]: #.*$\n/m   // [comment]: # xxx
+%ignore /^# .+$\n/m    // the line starting with "# "
+%ignore /^$\n/m        // empty lines
diff --git a/code/maintenance.py b/code/maintenance.py
index 61bddae3..dae21af8 100644
--- a/code/maintenance.py
+++ b/code/maintenance.py
@@ -14,6 +14,8 @@ import json
 import textwrap
 import os
 import re
+
+import utils.constants
 from utils import constants as c, utils, osg
 
 
@@ -76,7 +78,7 @@ def update_readme_and_tocs(infos):
 
     # create by category
     categories_text = []
-    for keyword in osg.recommended_keywords:
+    for keyword in utils.constants.recommended_keywords:
         infos_filtered = [x for x in infos if keyword in x['keywords']]
         title = keyword.capitalize()
         name = keyword.replace(' ', '-')
@@ -88,7 +90,7 @@ def update_readme_and_tocs(infos):
 
     # create by platform
     platforms_text = []
-    for platform in osg.valid_platforms:
+    for platform in utils.constants.valid_platforms:
         infos_filtered = [x for x in infos if platform in x.get('platform', [])]
         title = platform
         name = platform.lower()
@@ -271,7 +273,7 @@ def fix_entries():
         elements = list(set(elements))
 
         # get category out
-        for keyword in osg.recommended_keywords:
+        for keyword in utils.constants.recommended_keywords:
             if keyword in elements:
                 elements.remove(keyword)
                 category = keyword
@@ -949,12 +951,12 @@ def check_code_dependencies(infos):
     """
 
     # get all names of frameworks and library also using osg.code_dependencies_aliases
-    valid_dependencies = list(osg.code_dependencies_without_entry.keys())
+    valid_dependencies = list(utils.constants.general_code_dependencies_without_entry.keys())
     for info in infos:
         if any((x in ('framework', 'library', 'game engine') for x in info['keywords'])):
             name = info['name']
-            if name in osg.code_dependencies_aliases:
-                valid_dependencies.extend(osg.code_dependencies_aliases[name])
+            if name in utils.constants.code_dependencies_aliases:
+                valid_dependencies.extend(utils.constants.code_dependencies_aliases[name])
             else:
                 valid_dependencies.append(name)
 
diff --git a/code/maintenance_collect_inspiration_infos.py b/code/maintenance_collect_inspiration_infos.py
index 92288f94..5a5ed088 100644
--- a/code/maintenance_collect_inspiration_infos.py
+++ b/code/maintenance_collect_inspiration_infos.py
@@ -6,4 +6,4 @@ if __name__ == "__main__":
     osg.write_inspirations_info(inspirations)  # write again just to check integrity
 
     # assemble info
-    entries = osg.assemble_infos()
+    # entries = osg.assemble_infos()
diff --git a/code/utils/constants.py b/code/utils/constants.py
index bd24a5b7..39f29555 100644
--- a/code/utils/constants.py
+++ b/code/utils/constants.py
@@ -11,6 +11,10 @@ entries_path = os.path.join(root_path, 'entries')
 tocs_path = os.path.join(entries_path, 'tocs')
 code_path = os.path.join(root_path, 'code')
 
+inspirations_file = os.path.join(root_path, 'inspirations.md')
+developer_file = os.path.join(root_path, 'developer.md')
+
+# local config
 local_config_file = os.path.join(root_path, 'local-config.ini')
 
 config = configparser.ConfigParser()
@@ -24,3 +28,68 @@ def get_config(key):
     :return:
     """
     return config['general'][key]
+
+# database entry constants
+generic_comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
+
+# these fields have to be present in each entry (in this order)
+essential_fields = ('Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
+
+# only these fields can be used currently (in this order)
+valid_fields = (
+    'Home', 'Media', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
+    'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Build system', 'Build instructions')
+
+# these are the only valid platforms currently (and must be given in this order)
+valid_platforms = ('Windows', 'Linux', 'macOS', 'Android', 'iOS', 'Web')
+
+# at least one of these must be used for every entry, this gives the principal categories and the order of the categories
+recommended_keywords = (
+    'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation',
+    'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
+
+# known programming languages, anything else will result in a warning during a maintenance operation
+# only these will be used when gathering statistics
+known_languages = (
+    'AGS Script', 'ActionScript', 'Ada', 'AngelScript', 'Assembly', 'Basic', 'Blender Script', 'BlitzMax', 'C', 'C#',
+    'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript',
+    'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua',
+    'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R',
+    "Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
+    'Visual Basic', 'XUL', 'ZenScript', 'ooc')
+
+# known licenses, anything outside of this will result in a warning during a maintenance operation
+# only these will be used when gathering statistics
+known_licenses = (
+    '2-clause BSD', '3-clause BSD', 'AFL-3.0', 'AGPL-3.0', 'Apache-2.0', 'Artistic License-1.0', 'Artistic License-2.0',
+    'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0',
+    'CC-BY-SA-4.0', 'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0',
+    'LGPL-2.1', 'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License',
+    'None', 'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
+
+# valid multiplayer modes (can be combined with "+" )
+valid_multiplayer_modes = (
+    'competitive', 'co-op', 'hotseat', 'LAN', 'local', 'massive', 'matchmaking', 'online', 'split-screen')
+
+# TODO put the abbreviations directly in the name line (parenthesis maybe), that is more natural
+# this is a mapping of entry name to abbreviation and the abbreviations are used when specifying code dependencies
+code_dependencies_aliases = {'Simple DirectMedia Layer': ('SDL', 'SDL2'), 'Simple and Fast Multimedia Library': ('SFML',),
+                             'Boost (C++ Libraries)': ('Boost',), 'SGE Game Engine': ('SGE',), 'MegaGlest': ('MegaGlest Engine',)}
+
+# these are code dependencies that won't get their own entry, because they are not centered on gaming
+general_code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
+                                   'GLUT': 'https://www.opengl.org/resources/libraries/',
+                                   'WebGL': 'https://www.khronos.org/webgl/',
+                                   'Unity': 'https://unity.com/solutions/game',
+                                   '.NET': 'https://dotnet.microsoft.com/', 'Vulkan': 'https://www.khronos.org/vulkan/',
+                                   'KDE Frameworks': 'https://kde.org/products/frameworks/',
+                                   'jQuery': 'https://jquery.com/',
+                                   'node.js': 'https://nodejs.org/en/',
+                                   'GNU Guile': 'https://www.gnu.org/software/guile/',
+                                   'tkinter': 'https://docs.python.org/3/library/tk.html'}
+
+# developer information (in the file all fields will be capitalized)
+valid_developer_fields = ('name', 'games', 'contact', 'organization', 'home')
+
+# inspiration/original game information (in the file all fields will be capitalized)
+valid_inspiration_fields = ('name', 'inspired entries')
\ No newline at end of file
diff --git a/code/utils/osg.py b/code/utils/osg.py
index bc9278f6..5d08f546 100644
--- a/code/utils/osg.py
+++ b/code/utils/osg.py
@@ -3,32 +3,54 @@ Specific functions working on the games.
 """
 
 import re
-import os
 from difflib import SequenceMatcher
-from utils import utils, constants as c
+from utils import utils
 import lark
 
+from utils.constants import *
+
 
 class ListingTransformer(lark.Transformer):
+    """
+    Transforms content parsed by grammar_listing.lark further.
+    Used for the developer and inspirations list.
+    """
 
-    def number(self, x):
-        raise lark.Discard
+    def unquoted_value(self, x):
+        return x[0].value
+
+    def quoted_value(self, x):
+        return x[0].value[1:-1]  # remove quotation marks
 
     def property(self, x):
-        return x[0].value.lower(), x[1].value
+        """
+        The key of a property will be converted to lower case and the value part is the second part
+        :param x:
+        :return:
+        """
+        return x[0].lower(), x[1:]
 
     def name(self, x):
+        """
+        The name part is treated as a property with key "name"
+        :param x:
+        :return:
+        """
         return 'name', x[0].value
 
     def entry(self, x):
+        """
+        All (key, value) tuples are inserted into a dictionary.
+        :param x:
+        :return:
+        """
         d = {}
         for key, value in x:
+            if key in d:
+                raise RuntimeError('Key in entry appears twice')
             d[key] = value
         return d
 
-    def header(self, x):
-        raise lark.Discard
-
     def start(self, x):
         return x
 
@@ -61,53 +83,9 @@ class EntryTransformer(lark.Transformer):
         return 'building', d
 
 
-essential_fields = ('Home', 'State', 'Keywords', 'Code repository', 'Code language', 'Code license')
-valid_fields = (
-    'Home', 'Media', 'State', 'Play', 'Download', 'Platform', 'Keywords', 'Code repository', 'Code language',
-    'Code license', 'Code dependencies', 'Assets license', 'Developer', 'Build system', 'Build instructions')
-valid_platforms = ('Windows', 'Linux', 'macOS', 'Android', 'iOS', 'Web')
-recommended_keywords = (
-    'action', 'arcade', 'adventure', 'visual novel', 'sports', 'platform', 'puzzle', 'role playing', 'simulation',
-    'strategy', 'cards', 'board', 'music', 'educational', 'tool', 'game engine', 'framework', 'library', 'remake')
-known_languages = (
-    'AGS Script', 'ActionScript', 'Ada', 'AngelScript', 'Assembly', 'Basic', 'Blender Script', 'BlitzMax', 'C', 'C#',
-    'C++', 'Clojure', 'CoffeeScript', 'ColdFusion', 'D', 'DM', 'Dart', 'Dia', 'Elm', 'Emacs Lisp', 'F#', 'GDScript',
-    'Game Maker Script', 'Go', 'Groovy', 'Haskell', 'Haxe', 'Io', 'Java', 'JavaScript', 'Kotlin', 'Lisp', 'Lua',
-    'MegaGlest Script', 'MoonScript', 'None', 'OCaml', 'Objective-C', 'PHP', 'Pascal', 'Perl', 'Python', 'QuakeC', 'R',
-    "Ren'py", 'Ruby', 'Rust', 'Scala', 'Scheme', 'Script', 'Shell', 'Swift', 'TorqueScript', 'TypeScript', 'Vala',
-    'Visual Basic', 'XUL', 'ZenScript', 'ooc')
-known_licenses = (
-    '2-clause BSD', '3-clause BSD', 'AFL-3.0', 'AGPL-3.0', 'Apache-2.0', 'Artistic License-1.0', 'Artistic License-2.0',
-    'Boost-1.0', 'CC-BY-NC-3.0', 'CC-BY-NC-SA-2.0', 'CC-BY-NC-SA-3.0', 'CC-BY-SA-3.0', 'CC-BY-NC-SA-4.0',
-    'CC-BY-SA-4.0',
-    'CC0', 'Custom', 'EPL-2.0', 'GPL-2.0', 'GPL-3.0', 'IJG', 'ISC', 'Java Research License', 'LGPL-2.0', 'LGPL-2.1',
-    'LGPL-3.0', 'MAME', 'MIT', 'MPL-1.1', 'MPL-2.0', 'MS-PL', 'MS-RL', 'NetHack General Public License', 'None',
-    'Proprietary', 'Public domain', 'SWIG license', 'Unlicense', 'WTFPL', 'wxWindows license', 'zlib')
-known_multiplayer_modes = (
-    'competitive', 'co-op', 'hotseat', 'LAN', 'local', 'massive', 'matchmaking', 'online', 'split-screen')
-
-# TODO put the abbreviations directly in the name line (parenthesis maybe), that is more natural
-code_dependencies_aliases = {'Simple DirectMedia Layer': ('SDL', 'SDL2'), 'Simple and Fast Multimedia Library': ('SFML',),
-                             'Boost (C++ Libraries)': ('Boost',), 'SGE Game Engine': ('SGE',), 'MegaGlest': ('MegaGlest Engine',)}
-code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/',
-                                   'GLUT': 'https://www.opengl.org/resources/libraries/',
-                                   'WebGL': 'https://www.khronos.org/webgl/',
-                                   'Unity': 'https://unity.com/solutions/game',
-                                   '.NET': 'https://dotnet.microsoft.com/', 'Vulkan': 'https://www.khronos.org/vulkan/',
-                                   'KDE Frameworks': 'https://kde.org/products/frameworks/',
-                                   'jQuery': 'https://jquery.com/',
-                                   'node.js': 'https://nodejs.org/en/',
-                                   'GNU Guile': 'https://www.gnu.org/software/guile/',
-                                   'tkinter': 'https://docs.python.org/3/library/tk.html'}
-
 regex_sanitize_name = re.compile(r"[^A-Za-z 0-9-+]+")
 regex_sanitize_name_space_eater = re.compile(r" +")
 
-valid_developer_fields = ('name', 'games', 'contact', 'organization', 'home')
-valid_inspiration_fields = ('name', 'inspired entries')
-
-comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)'
-
 
 def name_similarity(a, b):
     return SequenceMatcher(None, str.casefold(a), str.casefold(b)).ratio()
@@ -130,11 +108,11 @@ def entry_iterator():
     """
 
     # get all entries (ignore everything starting with underscore)
-    entries = os.listdir(c.entries_path)
+    entries = os.listdir(entries_path)
 
     # iterate over all entries
     for entry in entries:
-        entry_path = os.path.join(c.entries_path, entry)
+        entry_path = os.path.join(entries_path, entry)
 
         # ignore directories ("tocs" for example)
         if os.path.isdir(entry_path):
@@ -350,8 +328,8 @@ def assemble_infos():
         # we also allow -X with X =2..9 as possible extension (because of duplicate canonical file names)
         if canonical_file_name != entry and canonical_file_name != entry[:-5] + '.md':
             print('Warning: file {} should be {}'.format(entry, canonical_file_name))
-            source_file = os.path.join(c.entries_path, entry)
-            target_file = os.path.join(c.entries_path, canonical_file_name)
+            source_file = os.path.join(entries_path, entry)
+            target_file = os.path.join(entries_path, canonical_file_name)
             if not os.path.isfile(target_file):
                 pass
                 # os.rename(source_file, target_file)
@@ -390,9 +368,10 @@ def extract_links():
     return urls
 
 
-def read_and_parse(content_file, grammar_file, transformer):
+def read_and_parse(content_file: str, grammar_file: str, transformer: lark.Transformer):
     """
-
+    Reads a content file and a grammar file and parses the content with the grammar following by
+    transforming the parsed output and returning the transformed result.
     :param content_file:
     :param grammar_file:
     :param transformer:
@@ -410,8 +389,7 @@ def read_developer_info():
 
     :return:
     """
-    developer_file = os.path.join(c.root_path, 'developer.md')
-    grammar_file = os.path.join(c.code_path, 'grammar_listing.lark')
+    grammar_file = os.path.join(code_path, 'grammar_listing.lark')
     transformer = ListingTransformer()
     developers = read_and_parse(developer_file, grammar_file, transformer)
     # now transform a bit more
@@ -446,7 +424,7 @@ def write_developer_info(developers):
     :return:
     """
     # comment
-    content = '{}\n'.format(comment_string)
+    content = '{}\n'.format(generic_comment_string)
 
     # number of developer
     content += '# Developer ({})\n\n'.format(len(developers))
@@ -474,22 +452,26 @@ def write_developer_info(developers):
         content += '\n'
 
     # write
-    developer_file = os.path.join(c.root_path, 'developer.md')
     utils.write_text(developer_file, content)
 
 
 def read_inspirations_info():
     """
-
+    Reads the info list about the games originals/inspirations from inspirations.md using the Lark parser grammar
+    in grammar_listing.lark
     :return:
     """
-    inspirations_file = os.path.join(c.root_path, 'inspirations.md')
-    grammar_file = os.path.join(c.code_path, 'grammar_listing.lark')
+    # read inspirations
+
+    grammar_file = os.path.join(code_path, 'grammar_listing.lark')
     transformer = ListingTransformer()
     inspirations = read_and_parse(inspirations_file, grammar_file, transformer)
+
+    # now inspirations is a list of dictionaries for every entry with keys (valid_developers_fields)
+
     # now transform a bit more
     for index, inspiration in enumerate(inspirations):
-        # check for valid keys
+        # check that keys are valid keys
         for field in inspiration.keys():
             if field not in valid_inspiration_fields:
                 raise RuntimeError('Unknown field "{}" for inspiration: {}.'.format(field, inspiration['name']))
@@ -497,26 +479,27 @@ def read_inspirations_info():
         for field in ('inspired entries',):
             if field in inspiration:
                 content = inspiration[field]
-                content = content.split(',')
                 content = [x.strip() for x in content]
                 inspiration[field] = content
+
     # check for duplicate names entries
     names = [inspiration['name'] for inspiration in inspirations]
     duplicate_names = (name for name in names if names.count(name) > 1)
     duplicate_names = set(duplicate_names)  # to avoid duplicates in duplicate_names
     if duplicate_names:
         print('Warning: duplicate inspiration names: {}'.format(', '.join(duplicate_names)))
+
     return inspirations
 
 
 def write_inspirations_info(inspirations):
     """
-
+    Given an internal list of inspirations, write it into the inspirations file
     :param inspirations:
     :return:
     """
     # comment
-    content = '{}\n'.format(comment_string)
+    content = '{}\n'.format(generic_comment_string)
 
     # number of developer
     content += '# Inspirations ({})\n\n'.format(len(inspirations))
@@ -545,7 +528,6 @@ def write_inspirations_info(inspirations):
         content += '\n'
 
     # write
-    inspirations_file = os.path.join(c.root_path, 'inspirations2.md')
     utils.write_text(inspirations_file, content)
 
 
diff --git a/inspirations.md b/inspirations.md
index c26747ea..561d9f48 100644
--- a/inspirations.md
+++ b/inspirations.md
@@ -31,7 +31,7 @@
 
 ## Achtung die Kurve! (3)
 
-- Inspired entries: Achtung, die Kurve!, Netacka, Zatacka X
+- Inspired entries: "Achtung, die Kurve!", Netacka, Zatacka X
 
 ## Advance Wars (1)
 
@@ -1311,7 +1311,7 @@
 
 ## RARS (1)
 
-- Inspired entries: TORCS, The Open Racing Car Simulator
+- Inspired entries: "The Open Racing Car Simulator, TORCS"
 
 ## Redneck Rampage (1)