From 48ade4bc0bc4ba4e2c12da30a426d763d2b76a1d Mon Sep 17 00:00:00 2001 From: Trilarion Date: Mon, 10 Feb 2020 12:35:22 +0100 Subject: [PATCH] adding developers --- code/grammar_listing.lark | 4 +- code/maintenance_collect_developer_infos.py | 129 ++++++++++++++++++++ code/requirements.txt | 3 +- code/utils/osg.py | 52 +++++++- code/utils/osg_github.py | 35 +++++- developer.md | 124 ++++++++++++++++++- entries/0_ad.md | 1 + entries/2048.md | 2 +- entries/2h4u.md | 2 +- entries/2moons_browsergame_engine.md | 2 +- entries/abes_amazing_adventure.md | 2 +- entries/advanced_strategic_command.md | 1 + entries/alex_the_allegator_4.md | 2 +- 13 files changed, 345 insertions(+), 14 deletions(-) create mode 100644 code/maintenance_collect_developer_infos.py diff --git a/code/grammar_listing.lark b/code/grammar_listing.lark index feab3b1d..baa784de 100644 --- a/code/grammar_listing.lark +++ b/code/grammar_listing.lark @@ -8,10 +8,10 @@ property: "- " _key ": " _value "\n" _key: /(?! ).+?(?=:)(? 40: + break + + # print + entry_name = '{} - {}'.format(entry['file'], entry['name']) + print(entry_name) + content = '' + + entry_developer = entry.get('developer', []) + + # parse home + home = entry['home'] + # sourceforge project site + prefix = 'https://sourceforge.net/projects/' + url = [x for x in home if x.startswith(prefix)] + if len(url) == 1: + url = url[0] + print(' sourceforge project site: {}'.format(url)) + url = 'https://sourceforge.net/p/' + url[len(prefix):] + '_members/' + response = requests.get(url) + soup = BeautifulSoup(response.text, 'html.parser') + authors = soup.find('div', id='content_base').find('table').find_all('tr') + authors = [author.find_all('td') for author in authors] + authors = [author[1].a['href'] for author in authors if len(author) == 3] + for author in authors: + # sometimes author already contains the full url, sometimes not + url = 'https://sourceforge.net' + author if not author.startswith('http') else author + response = requests.get(url) + url = response.url # could be different now + if 'auth/?return_to' in url: + # for some reason authorisation is forbidden + author_name = author + nickname = author + else: + soup = BeautifulSoup(response.text, 'html.parser') + author_name = soup.h1.get_text() + author_name = SF_alias_list.get(author_name, author_name) # replace by alias if possible + nickname = soup.find('dl', class_= 'personal-data').find('dd').get_text() + nickname = nickname.replace('\n', '').strip() + dev = developer_info_lookup(author_name) + in_devs = dev and 'contact' in dev and nickname + '@SF' in dev['contact'] + in_entry = author_name in entry_developer + if in_devs and in_entry: + continue # already existing in entry and devs + content += ' {} : {}@SF'.format(author_name, nickname) + if not in_devs: + content += ' (not in devs)' + if not in_entry: + content += ' (not in entry)' + content += '\n' + + # parse source repository + repos = entry.get('code repository', []) + + # Github + urls = [x for x in repos if x.startswith('https://github.com/')] + urls = [] + for url in urls: + print(' github repo: {}'.format(url)) + github_info = osg_github.retrieve_repo_info(url) + for contributor in github_info['contributors']: + name = contributor.name + dev = developer_info_lookup(name) + in_devs = dev and 'contact' in dev and contributor.login + '@GH' in dev['contact'] + in_entry = name in entry_developer + if in_devs and in_entry: + continue # already existing in entry and devs + content += ' {}: {}@GH'.format(name, contributor.login) + if contributor.blog: + content += ' url: {}'.format(contributor.blog) + if not in_devs: + content += ' (not in devs)' + if not in_entry: + content += ' (not in entry)' + content += '\n' + + if content: + developers += '{}\n\n{}\n'.format(entry_name, content) + + + except RuntimeError as e: + raise(e) + # pass + finally: + # store developer info + utils.write_text(os.path.join(c.root_path, 'collected_developer_info.txt'), developers) \ No newline at end of file diff --git a/code/requirements.txt b/code/requirements.txt index a2fc6ce2..6bc9b5f4 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -1,2 +1,3 @@ pygithub -lark-parser \ No newline at end of file +lark-parser +BeautifulSoup \ No newline at end of file diff --git a/code/utils/osg.py b/code/utils/osg.py index 3abaf61a..ecc8f7f6 100644 --- a/code/utils/osg.py +++ b/code/utils/osg.py @@ -15,7 +15,7 @@ class ListingTransformer(lark.Transformer): raise lark.Discard def property(self, x): - return (x[0].value, x[1].value) + return (x[0].value.lower(), x[1].value) def name(self, x): return ('name', x[0].value) @@ -71,6 +71,8 @@ code_dependencies_without_entry = {'OpenGL': 'https://www.opengl.org/', regex_sanitize_name = re.compile(r"[^A-Za-z 0-9-+]+") regex_sanitize_name_space_eater = re.compile(r" +") +valid_developer_fields = ('name', 'games', 'contact', 'organization', 'home') + comment_string = '[comment]: # (partly autogenerated content, edit with care, read the manual before)' @@ -378,7 +380,22 @@ def read_developer_info(): developer_file = os.path.join(c.root_path, 'developer.md') grammar_file = os.path.join(c.code_path, 'grammar_listing.lark') transformer = ListingTransformer() - return read_and_parse(developer_file, grammar_file, transformer) + developers = read_and_parse(developer_file, grammar_file, transformer) + # now transform a bit more + for index, dev in enumerate(developers): + for field in dev.keys(): + if field not in valid_developer_fields: + raise RuntimeError('Unknown developer field "{}" for developer: {}.'.format(field, dev['name'])) + for field in ('name', 'organization'): + if field in dev: + dev[field] = dev[field].strip() + for field in ('games', 'contact'): + if field in dev: + content = dev[field] + content = content.split(',') + content = [x.strip() for x in content] + dev[field] = content + return developers def write_developer_info(developers): @@ -386,7 +403,38 @@ def write_developer_info(developers): :return: """ + # comment + content = '{}\n'.format(comment_string) + + # number of developer + content += '# Developer ({})\n\n'.format(len(developers)) + + # sort by name + developers.sort(key=lambda x: str.casefold(x['name'])) + + # iterate over them + for dev in developers: + # developer name + content += '## {} ({})\n\n'.format(dev['name'], len(dev['games'])) + + # games + content += '- Games: {}\n'.format(', '.join(sorted(dev['games'], key=str.casefold))) + + # all the remaining in alphabetical order + for field in sorted(dev.keys()): + if field not in ('name', 'games'): + value = dev[field] + field = field.capitalize() + if isinstance(value, str): + content += '- {}: {}\n'.format(field, value) + else: + content += '- {}: {}\n'.format(field, ', '.join(sorted(value, key=str.casefold))) + content += '\n' + + # write developer_file = os.path.join(c.root_path, 'developer.md') + utils.write_text(developer_file, content) + def read_inspirations_info(): diff --git a/code/utils/osg_github.py b/code/utils/osg_github.py index 11c4152b..e4c5cbf4 100644 --- a/code/utils/osg_github.py +++ b/code/utils/osg_github.py @@ -5,16 +5,45 @@ Everything specific to the Github API (via PyGithub). from github import Github +def normalize_repo_name(repo): + """ + Bring repo to style xxx/yyy + """ + prefix = 'https://github.com/' + if repo.startswith(prefix): + repo = repo[len(prefix):] + suffix = '.git' + if repo.endswith(suffix): + repo = repo[:-len(suffix)] + return repo + + +def repo_get_contributors(repo): + contributors = [] + c = repo.get_contributors() + for i in range(c.totalCount): + contributors.append(c[i]) + return contributors + + def retrieve_repo_info(repos): """ - For a list of Github repos, retrieves repo information + For a list of Github repos, retrieves repo information. + + Repos must be have the style xxx/yyy example: "PyGithub/PyGithub" """ + single_repo = isinstance(repos, str) + if single_repo: + repos = (repos,) result = [] g = Github() for repo in repos: + repo = normalize_repo_name(repo) r = g.get_repo(repo) - e = {'archived': r.archived, 'description': r.description, 'language': r.language, - 'last modified': r.last_modified, 'open issues count': r.open_issues_count, + e = {'archived': r.archived, 'contributors': repo_get_contributors(r), 'description': r.description, + 'language': r.language, 'last modified': r.last_modified, 'open issues count': r.open_issues_count, 'stars count': r.stargazers_count, 'topics': r.topics, 'repo': repo} result.append(e) + if single_repo: + result = result[0] return result diff --git a/developer.md b/developer.md index 237bb67e..fd724f99 100644 --- a/developer.md +++ b/developer.md @@ -79,6 +79,7 @@ ## Ari Mustonen (1) - Games: Hex-a-hop +- Contact: amuzen@SF ## Arne Reiners (1) @@ -137,6 +138,12 @@ - Games: XBill +## Ben Brian (1) + +- Games: 0 A.D. +- Home: https://benbrian.net/ +- Contact: historic_bruno@SF, historicbruno@GH + ## Bruno Ethvignot (2) - Games: Powermanga, TecnoballZ @@ -199,6 +206,21 @@ - Games: Colobot: Gold Edition - Organization: Epsitec +## Nicolas Auvray (1) + +- Games: 0 A.D. +- Contact: itms@SF, na-Itms@GH + +## Philip Taylor (1) + +- Games: 0 A.D. +- Contact: philiptaylor@SF, philiptaylor@GH + +## Lancelot de Ferrière (1) + +- Games: 0 A.D. +- Contact: wraitii@SF, wraitii@GH + ## Daniele Napolitano (1) - Games: Gweled @@ -450,6 +472,31 @@ - Games: Fairy-Max +## Dorfdrull (1) + +- Games: Advanced Strategic Command +- Contact: dorfdrull@SF + +## Michael Moerz (1) + +- Games: Advanced Strategic Command +- Contact: natoka@SF + +## Armin Bajramovic (1) + +- Games: Advanced Strategic Command +- Contact: armin906@SF + +## Frederik Kesting (1) + +- Games: Advanced Strategic Command +- Contact: ocl4nis@SF + +## Torsten Maekler (1) + +- Games: Advanced Strategic Command +- Contact: tmaekler@SF + ## Harmen van der Wal (1) - Games: Hypercube @@ -544,7 +591,7 @@ ## Jason Rohrer (8) - Games: Between, Cultivation, Gravitation, One Hour One Life, Passage, Primrose, Sleep Is Death, Transcend -- Contact: jcr13@SF +- Contact: jcr13@SF, jasonrohrer@GH ## Jay Fenlason (1) @@ -612,6 +659,12 @@ ## Johan Peitz (1) - Games: Alex the Allegator 4 +- Contact: peitz@SF + +## Erik Johansson (1) + +- Games: 0 A.D. +- Contact: feneur@SF ## Johannes Bergmeier (1) @@ -748,6 +801,12 @@ - Games: TecnoballZ +## Laurent (1) + +- Games: 2048 +- Home: https://uto.io/ +- Contact: marg51@GH + ## legoluft (1) - Games: Krank @@ -827,6 +886,21 @@ - Games: Holtz +## Martin Bickel (1) + +- Games: Advanced Strategic Command +- Contact: valharis@SF, ValHaris@GH + +## Christian Schramm (1) + +- Games: Advanced Strategic Command +- Contact: Ed-von-Schleck@GH + +## valuial (1) + +- Games: Advanced Strategic Command +- Contact: valuial@GH + ## Masanao Izumo (1) - Games: Kobo Deluxe @@ -1033,6 +1107,7 @@ ## Paul Wise (1) - Games: Hex-a-hop +- Contact: pabs3@SF ## Pete Shinners (1) @@ -1073,6 +1148,11 @@ - Games: Trip on the Funny Boat +## Pureon (1) + +- Games: 0 A.D. +- Contact: ipureon@SF + ## PyMike (1) - Games: Mrfuze @@ -1193,6 +1273,11 @@ - Games: Freya Game Engine - Contact: pond@SF +## Kayl (1) + +- Games: 2H4U +- Contact: kaylnet@SF + ## Shard (1) - Games: Anagramarama @@ -1217,6 +1302,33 @@ ## Simon Laszlo (1) - Games: 4D-TRIS +- Contact: simzer@SF + +## s0600204 (1) + +- Games: 0 A.D. +- Contact: s0600204@GH + +## Hilarious001 (1) + +- Games: 2Moons Browsergame Engine +- Contact: Hilarious001@GH + +## Ozan Kurt (1) + +- Games: 2Moons Browsergame Engine +- Contact: OzanKurt@GH + +## leper (1) + +- Games: 0 A.D. +- Contact: leper@GH + +## Kieran Pilkington (1) + +- Games: 0 A.D. +- Home: https://k776.tumblr.com/ +- Contact: KieranP@GH ## Simon Peter (1) @@ -1384,6 +1496,16 @@ - Games: Qonk +## Alex Clark (1) + +- Games: Abe's Amazing Adventure +- Contact: jazkat@SF + +## Pedro Izecksohn (1) + +- Games: Abe's Amazing Adventure +- Contact: izecksohn@SF + ## Tuscan Knox (1) - Games: Shotgun Debugger diff --git a/entries/0_ad.md b/entries/0_ad.md index 41ff212e..60d2d92f 100644 --- a/entries/0_ad.md +++ b/entries/0_ad.md @@ -13,6 +13,7 @@ _0 A.D. is a free, open-source, cross-platform real-time strategy game._ - Code license: GPL-2.0 - Code dependencies: libogg, libvorbis, libxml2, OpenAL, SDL2, zlib - Assets license: CC-BY-SA-3.0 +- Developer: Erik Johansson, Ben Brian, Nicolas Auvray, Philip Taylor, Lancelot de Ferrière, s0600204, Kieran Pilkington, leper, Pureon Engine part is called Pyrogenesis. diff --git a/entries/2048.md b/entries/2048.md index e06a552f..eeefbd1c 100644 --- a/entries/2048.md +++ b/entries/2048.md @@ -11,7 +11,7 @@ _Sliding block puzzle game._ - Code language: JavaScript - Code license: MIT - Assets license: MIT (very few assets) -- Developer: Gabriele Cirulli, Jerry Jiang (Android port) +- Developer: Gabriele Cirulli, Laurent, Jerry Jiang (Android port) Port to Android: https://github.com/tpcstld/2048 diff --git a/entries/2h4u.md b/entries/2h4u.md index 96b4ec01..a8be9351 100644 --- a/entries/2h4u.md +++ b/entries/2h4u.md @@ -10,6 +10,6 @@ _Mix between a Tetris-like game and a wall breaker._ - Code language: C++ - Code license: GPL-2.0 - Code dependencies: SDL -- Developer: Piwai +- Developer: Piwai, Kayl ## Building diff --git a/entries/2moons_browsergame_engine.md b/entries/2moons_browsergame_engine.md index fee28ef8..36f68c25 100644 --- a/entries/2moons_browsergame_engine.md +++ b/entries/2moons_browsergame_engine.md @@ -10,7 +10,7 @@ _Space browsergame framework._ - Code repository: https://github.com/jkroepke/2Moons.git (archived), https://github.com/steemnova/steemnova.git (+) - Code language: PHP, JavaScript - Code license: MIT -- Developer: Jan-Otto Kröpke +- Developer: Jan-Otto Kröpke, Ozan Kurt ## Building diff --git a/entries/abes_amazing_adventure.md b/entries/abes_amazing_adventure.md index 94b7e8dc..408dc72b 100644 --- a/entries/abes_amazing_adventure.md +++ b/entries/abes_amazing_adventure.md @@ -11,7 +11,7 @@ _Side scrolling platform game._ - Code language: C - Code license: GPL-2.0 - Code dependencies: SDL -- Developer: Gabor Torok +- Developer: Gabor Torok, Pedro Izecksohn, Alex Clark ## Building diff --git a/entries/advanced_strategic_command.md b/entries/advanced_strategic_command.md index 616576e7..bfe9d715 100644 --- a/entries/advanced_strategic_command.md +++ b/entries/advanced_strategic_command.md @@ -9,6 +9,7 @@ _Turn based strategy game._ - Code repository: https://github.com/ValHaris/asc-hq.git - Code language: C++ - Code license: GPL-2.0 +- Developer: Martin Bickel, Christian Schramm, valuial, Armin Bajramovic, Dorfdrull, Michael Moerz, Frederik Kesting, Torsten Maekler ## Building diff --git a/entries/alex_the_allegator_4.md b/entries/alex_the_allegator_4.md index d67226f4..6d5cb175 100644 --- a/entries/alex_the_allegator_4.md +++ b/entries/alex_the_allegator_4.md @@ -12,7 +12,7 @@ _Retro-style platformer._ - Code license: GPL-2.0 - Code dependencies: Allegro - Assets license: GPL-2.0 -- Developer: Johan Peitz (design source code and graphics), Anders Svensson (music and sound effects) +- Developer: Johan Peitz (design source code and graphics), Anders Svensson (music and sound effects), Paul Wise ## Building