imports from libregamewiki

2019-10-23 13:44:58 +02:00
parent 206acda8a7
commit a2c6294b54
61 changed files with 1055 additions and 136 deletions
--- a/tools/archives.json
+++ b/tools/archives.json
@ -20,11 +20,16 @@
  "https://git.code.sf.net/p/autorealm/code",
  "https://git.code.sf.net/p/autorealm/delphi",
  "https://git.code.sf.net/p/autorealm/http",
+  "https://git.code.sf.net/p/biogenesis/git",
+  "https://git.code.sf.net/p/blobandconquer/code",
  "https://git.code.sf.net/p/blobwars/code",
  "https://git.code.sf.net/p/bygfoot/git",
+  "https://git.code.sf.net/p/caphgame/code",
  "https://git.code.sf.net/p/celestron/code",
  "https://git.code.sf.net/p/childsplay/git",
  "https://git.code.sf.net/p/chromium-bsu/code",
+  "https://git.code.sf.net/p/civitas/code",
+  "https://git.code.sf.net/p/csp/git",
  "https://git.code.sf.net/p/dangerdeep/git",
  "https://git.code.sf.net/p/deng/code",
  "https://git.code.sf.net/p/dnt/code",
@ -85,6 +90,7 @@
  "https://git.savannah.nongnu.org/git/lordsawar.git",
  "https://git.tukaani.org/xz.git",
  "https://git.tuxfamily.org/baller/baller.git",
+  "https://git.tuxfamily.org/boohu/boohu.git",
  "https://git.tuxfamily.org/fanwor/fanwor.git",
  "https://github.com/0ad/0ad.git",
  "https://github.com/AdamAtomic/flixel.git",
@ -113,6 +119,8 @@
  "https://github.com/BZFlag-Dev/bzflag.git",
  "https://github.com/BackupTheBerlios/netpanzer-svn.git",
  "https://github.com/Bacon2D/Bacon2D.git",
+  "https://github.com/Blackvoxel/Blackvoxel.git",
+  "https://github.com/Blasphemer/blasphemer.git",
  "https://github.com/BlindMindStudios/StarRuler2-Source.git",
  "https://github.com/Blzut3/CatacombSDL.git",
  "https://github.com/BrooklynKing/Grimsonland.git",
@ -123,12 +131,14 @@
  "https://github.com/CartBlanche/Siege-of-Avalon-Open-Source.git",
  "https://github.com/Ceva24/ceva24.github.io.git",
  "https://github.com/ChaosForge/doomrl.git",
+  "https://github.com/ChildrenOfUr/coUclient.git",
  "https://github.com/Chris3606/GoRogue.git",
  "https://github.com/ChrisDill/Raylib-cs.git",
  "https://github.com/Circular-Studios/Dash.git",
  "https://github.com/Cirrus-Minor/witchblast.git",
  "https://github.com/CleverRaven/Cataclysm-DDA.git",
  "https://github.com/CompletelyFairGames/dwarfcorp.git",
+  "https://github.com/Construo/construo.git",
  "https://github.com/CorsixTH/CorsixTH.git",
  "https://github.com/DCurrent/openbor.git",
  "https://github.com/DaemonEngine/Daemon.git",
@ -158,6 +168,7 @@
  "https://github.com/FreeFalcon/freefalcon-central.git",
  "https://github.com/FreeRCT/FreeRCT.git",
  "https://github.com/FreezingMoon/AncientBeast.git",
+  "https://github.com/FrenchYann/Chess3D.git",
  "https://github.com/Fundynamic/dune2themaker4j.git",
  "https://github.com/GNOME/aisleriot.git",
  "https://github.com/GNOME/atomix.git",
@ -189,6 +200,7 @@
  "https://github.com/JACoders/OpenJK.git",
  "https://github.com/Jakz/openmom.git",
  "https://github.com/Javanaise/mrboom-libretro.git",
+  "https://github.com/KDE/blinken.git",
  "https://github.com/Kerndog73/EnTT-Pacman.git",
  "https://github.com/Kromster80/kam_remake.git",
  "https://github.com/LIKO-12/LIKO-12.git",
@ -324,6 +336,7 @@
  "https://github.com/Zillode/Slot-Racers.git",
  "https://github.com/Zukero/andors-trail.git",
  "https://github.com/a-nikolaev/curseofwar.git",
+  "https://github.com/abagames/consomaton.git",
  "https://github.com/abhinandanramesh/Star-Wars-III.git",
  "https://github.com/aburch/simutrans.git",
  "https://github.com/acmepjz/meandmyshadow.git",
@ -344,6 +357,7 @@
  "https://github.com/amerkoleci/vortice.git",
  "https://github.com/amroibrahim/DIYDoom.git",
  "https://github.com/anael-seghezzi/Maratis-4.git",
+  "https://github.com/anaseto/boohu.git",
  "https://github.com/andrettin/wyrmsun.git",
  "https://github.com/andrewfenn/Hardwar.git",
  "https://github.com/andwj/torrega.git",
@ -375,6 +389,7 @@
  "https://github.com/bibendovsky/bstone.git",
  "https://github.com/bitcraft/PyTMX.git",
  "https://github.com/bitcraft/pyscroll.git",
+  "https://github.com/bitfighter/bitfighter.git",
  "https://github.com/bladecoder/bladecoder-adventure-engine.git",
  "https://github.com/blakeohare/pyweek-sentientstorage.git",
  "https://github.com/blockattack/blockattack-game.git",
@ -469,6 +484,7 @@
  "https://github.com/fabiensanglard/chocolate_duke3D.git",
  "https://github.com/falltergeist/falltergeist.git",
  "https://github.com/fariazz/World-of-Heroes.git",
+  "https://github.com/farsides/cardstories.git",
  "https://github.com/fasterthanlime/isaac-paper.git",
  "https://github.com/fastrgv/AdaVenture.git",
  "https://github.com/fifengine/fifengine.git",
@ -554,6 +570,7 @@
  "https://github.com/jpmalkiewicz/decker.git",
  "https://github.com/jsettlers/settlers-remake.git",
  "https://github.com/jtc0de/Blitwizard.git",
+  "https://github.com/jtrulson/conquest.git",
  "https://github.com/juanjepl/polisgame.git",
  "https://github.com/juzzlin/DustRacing2D.git",
  "https://github.com/jwm-art-net/XorCurses.git",
@ -737,6 +754,7 @@
  "https://github.com/soap-bubble/web.git",
  "https://github.com/soulweaver91/project-carrot.git",
  "https://github.com/sphair/ClanLib.git",
+  "https://github.com/spiderbit/canta.git",
  "https://github.com/spleennooname/shadow-of-the-beast-html5.git",
  "https://github.com/spring/spring.git",
  "https://github.com/spring1944/spring1944.git",
@ -786,6 +804,7 @@
  "https://github.com/trackballs/trackballs.git",
  "https://github.com/trananh1992/Turn-of-War.git",
  "https://github.com/triplea-game/triplea.git",
+  "https://github.com/tsadok/brogue.git",
  "https://github.com/ttdpatch/ttdpatch.git",
  "https://github.com/ttyborg/erampage.git",
  "https://github.com/ufo2000/ufo2000.git",
@ -841,6 +860,7 @@
  "https://gitlab.com/KilgoreTroutMaskReplicant/1oom.git",
  "https://gitlab.com/LibreGames/freesiege.git",
  "https://gitlab.com/SuperTuxParty/SuperTuxParty.git",
+  "https://gitlab.com/bjk/cboard.git",
  "https://gitlab.com/drummyfish/Bombman.git",
  "https://gitlab.com/evol/evol-all.git",
  "https://gitlab.com/freedroid/freedroid-src.git",
@ -1011,6 +1031,7 @@
  "https://gitlab.openrsc.com/open-rsc/Game.git",
  "https://gitlab.openrsc.com/open-rsc/Single-Player.git",
  "https://gitorious.org/turious/turious.git",
+  "https://salsa.debian.org/games-team/bloboats.git",
  "https://src.fedoraproject.org/rpms/blobwars.git"
 ],
 "svn": [
--- a/tools/backlog.txt
+++ b/tools/backlog.txt
@ -1,3 +1,4 @@
+Glitch the game, by Tiny Speck Inc. public domain
 http://antongerdelan.net/blog/ (other projects besides TestDrive)
 http://circularstudios.com/
 http://cyxdown.free.fr/bs/
@ -118,6 +119,7 @@ https://archive.codeplex.com/?p=turnota
 https://archive.codeplex.com/?p=voxeliq
 https://blenderartists.org/t/devils-pinball/552785
 https://blends.debian.org/games/tasks/
+https://blends.debian.org/games/tasks/racing
 https://carrot.soulweaver.fi/#links
 https://cocainediesel.fun/
 https://code.google.com/archive/p/galacticthrone/
@ -127,6 +129,7 @@ https://code.google.com/archive/p/tecbattle/
 https://code.launchpad.net/~flosoft/s25rttr/trunk
 https://conquertheworldbycode.wordpress.com/nostalgia-2/
 https://dragengine.rptd.ch/
+https://edu.kde.org/
 https://empiredirectory.net/
 https://empiredirectory.net/index.php/downloads/viewdownload/6-server-software/13-empire-server
 https://en.wikipedia.org/w/index.php?title=GNU_Backgammon&action=edit&redlink=1
@ -154,12 +157,14 @@ https://flathub.org/home (use it for Linux packaging) / https://flathub.org/apps
 https://forums.scummvm.org/viewtopic.php?t=13512&highlight=open+source
 https://freegamer.blogspot.com (maybe there is something interesting)
 https://futurepinball.com/
+https://fydo.net/projects/island-rescue (and other projects there)
 https://gamejolt.com/ (search there)
 https://gdevelop-app.com/
 https://github.com/00-Evan/shattered-pixel-dungeon
 https://github.com/agateau/pixelwheels
 https://github.com/amerkoleci/Vortice.Windows
 https://github.com/ao-libre/ao-cliente
+https://github.com/AtomicGameEngine/AtomicGameEngine
 https://github.com/atphalix/nexuiz
 https://github.com/azhirnov/FrameGraph
 https://github.com/bioglaze/aether3d
@ -202,6 +207,8 @@ https://github.com/id-Software
 https://github.com/Illation/ETEngine
 https://github.com/JohanLi/uncharted-waters-2
 https://github.com/junkdog/artemis-odb
+https://github.com/KDE
+https://github.com/Kosmonaut3d/DeferredEngine
 https://github.com/LgLinus/StrategyGame
 https://github.com/libretro/libretro-prboom
 https://github.com/ligurio/awesome-ttygames
@ -237,6 +244,8 @@ https://github.com/skypjack/entt
 https://github.com/smlinux/nexuiz
 https://github.com/SPC-Some-Polish-Coders/PopHead
 https://github.com/SPC-Some-Polish-Coders/PopHead/
+https://github.com/stackos/Viry3D
+https://github.com/tainicom/Aether.Physics2D
 https://github.com/the-insulines
 https://github.com/theaigames
 https://github.com/Tinob/Ishiiruka (https://github.com/shiiion/Ishiiruka, https://github.com/SirMangler/PrimeHack-Updater)
--- a/tools/improvement_ideas.txt
+++ b/tools/improvement_ideas.txt
@ -1,4 +1,5 @@
 Star-Wars-III: convert to JavaScript
 Trackballs: adapt to Windows
 TUSSLE: Maybe some characters can be included by default
-Unchartered Waters: Was removed, but sounds interesting. https://koei.fandom.com/wiki/Uncharted_Waters:_New_Horizons
+Unchartered Waters: Was removed, but sounds interesting. https://koei.fandom.com/wiki/Uncharted_Waters:_New_Horizons, has been added again
+CannonSmash: Table tennis game
--- a/tools/libregamewiki_import.py
+++ b/tools/libregamewiki_import.py
@ -71,7 +71,8 @@ def parse_lgw_content():
    files = os.listdir(import_path)
    entries = []
    for file in files:
-        if file == '_lgw.json':
+        file = files[56]
+        if file.startswith('_lgw'):
            continue

        text = utils.read_text(os.path.join(import_path, file))
@ -83,8 +84,9 @@ def parse_lgw_content():
        entry = {'name': title}

        # get all external links
+        ignored_external_links = ('libregamewiki.org', 'freegamedev.net', 'freegamer.blogspot.com', 'opengameart.org', 'gnu.org', 'creativecommons.org', 'freesound.org', 'freecode.com')
        links = [(x['href'], x.get_text()) for x in soup.find_all('a', href=True)]
-        links = [x for x in links if x[0].startswith('http') and not x[0].startswith('https://libregamewiki.org/')]
+        links = [x for x in links if x[0].startswith('http') and not any([y in x[0] for y in ignored_external_links])]
        entry['external links'] = links

        # get meta description
@ -296,18 +298,18 @@ def clean_lgw_content():
    entries = remove_parenthized_content(entries, ('assets license', 'code language', 'code license', 'engine', 'genre', 'last active', 'library'))
    entries = remove_prefix_suffix(entries, ('code license', 'assets license'), ('"', 'GNU', ), ('"', '[3]', '[2]', '[1]', 'only'))
    entries = replace_content(entries, ('code license', 'assets license'), 'GPL', ('General Public License', ))
-    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv2', ('GPL v2', 'GPL version 2.0', 'GPL 2.0'))
+    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv2', ('GPL v2', 'GPL version 2.0', 'GPL 2.0', 'General Public License v2', 'GPL version 2', 'Gplv2', 'GPL 2'))
    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv2+', ('GPL v2 or later', 'GPL 2+', 'GPL v2+', 'GPL version 2 or later'))
    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv3', ('GPL v3', 'GNU GPL v3', 'GPL 3'))
-    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv3+', ('GPL v3+', 'GPL v.3 or later'))
+    entries = replace_content(entries, ('code license', 'assets license'), 'GPLv3+', ('GPL v3+', 'GPL v.3 or later', 'GPL v3 or later'))
    entries = replace_content(entries, ('code license', 'assets license'), 'Public domain', ('public domain', 'Public Domain'))
-    entries = replace_content(entries, ('code license', 'assets license'), 'zlib', ('zlib/libpng license', ))
+    entries = replace_content(entries, ('code license', 'assets license'), 'zlib', ('zlib/libpng license', 'Zlib License'))
    entries = replace_content(entries, ('code license', 'assets license'), 'BSD', ('Original BSD License', ))
    entries = replace_content(entries, ('code license', 'assets license'), 'CC-BY-SA-3.0', ('Creative Commons Attribution-ShareAlike 3.0 Unported License', 'CC-BY-SA 3.0', 'CC BY-SA 3.0'))
    entries = replace_content(entries, ('code license', 'assets license'), 'CC-BY-SA', ('CC BY-SA',))
-    entries = replace_content(entries, ('code license', 'assets license'), 'MIT', ('MIT License',))
+    entries = replace_content(entries, ('code license', 'assets license'), 'MIT', ('MIT License', 'MIT"'))
    entries = replace_content(entries, 'platform', 'macOS', ('Mac', ))
-    entries = remove_prefix_suffix(entries, 'code language', (), ('[3]', '[2]', '[1]'))
+    entries = remove_prefix_suffix(entries, ('code language', 'developer'), (), ('[3]', '[2]', '[1]'))
    entries = ignore_content(entries, 'code language', ('HTML5', 'HTML', 'English', 'XML', 'WML'))
    entries = replace_content(entries, 'code language', 'Lua', ('lua', 'LUA'))
    entries = remove_prefix_suffix(entries, 'genre', (), ('game', 'games'))
@ -325,6 +327,7 @@ def clean_lgw_content():
    entries = ignore_content(entries, 'last active', ('2019', ))
    entries = ignore_content(entries, 'platform', ('DOS', ))

+
    # list for every unique field
    # fields = sorted(list(unique_fields))
    fields = sorted(list(unique_fields - set(('description', 'external links', 'dev home', 'forum', 'home', 'linux-packages', 'developer', 'chat', 'tracker', 'Latest release', 'name', 'repo', 'Release date', 'categories'))))
@ -351,7 +354,7 @@ if __name__ == "__main__":
    # download_lgw_content()

    # stage two
-    # parse_lgw_content()
+    parse_lgw_content()

    # stage three
-    clean_lgw_content()
+    # clean_lgw_content()
--- a/tools/libregamewiki_synchronization.py
+++ b/tools/libregamewiki_synchronization.py
@ -29,9 +29,12 @@ from utils import constants, utils, osg

 name_replacements = {'Eat the Whistle': 'Eat The Whistle', 'Scorched 3D': 'Scorched3D', 'Silver Tree': 'SilverTree', 'Blob Wars Episode 1 : Metal Blob Solid': 'Blobwars: Metal Blob Solid', 'Adventure': 'Colossal Cave Adventure',
                     'Fall Of Imiryn': 'Fall of Imiryn', 'Liquid War 6': 'Liquid War', 'Gusanos': 'GUSANOS', 'Corewars': 'Core War', 'FLARE': 'Flare', 'Vitetris': 'vitetris', 'Powder Toy': 'The Powder Toy', 'Asylum': 'SDL Asylum',
-                     'Atanks': 'Atomic Tanks', 'HeXon': 'heXon', 'Unnethack': 'UnNetHack', 'Nova Pinball': 'NOVA PINBALL', 'Jump n Bump': "Jump'n'Bump"}
-ignored_names = ['Hetris', '8 Kingdoms', 'Antigravitaattori', 'Arena of Honour', 'Arkhart', 'Ascent of Justice', 'Balazar III', 'Balder3D', 'Barbie Seahorse Adventures', 'Barrage', 'Gnome Batalla Naval', 'User:AVRS/sandbox']
+                     'Atanks': 'Atomic Tanks', 'HeXon': 'heXon', 'Unnethack': 'UnNetHack', 'Nova Pinball': 'NOVA PINBALL', 'Jump n Bump': "Jump'n'Bump", 'Blades of Exile': 'Classic Blades of Exile', 'BlinKen': 'Blinken',
+                     'Colobot': 'Colobot: Gold Edition'}
+ignored_names = ['Hetris', '8 Kingdoms', 'Antigravitaattori', 'Arena of Honour', 'Arkhart', 'Ascent of Justice', 'Balazar III', 'Balder3D', 'Barbie Seahorse Adventures', 'Barrage', 'Gnome Batalla Naval', 'User:AVRS/sandbox', 'Blocks',
+                 'Brickshooter', 'Bweakfwu', 'Cheese Boys', 'Clippers', 'Codewars', 'CRAFT: The Vicious Vikings']

+licenses_map = {'GPLv2': 'GPL-2.0', 'GPLv2+': 'GPL-2.0', 'GPLv3': 'GPL-3.0', 'GPLv3+': 'GPL-3.0'}

 def list_compare(a, b, k):
    """
@ -161,17 +164,19 @@ if __name__ == "__main__":
            entry = '# {}\n\n'.format(lgw_name)

            # add empty description
-            entry += '__\n\n'
+            entry += '_{}_\n\n'.format(lgw_entry['description'])

            # empty home (mandatory on our side)
-            entry += '- Home: \n'
+            home = lgw_entry.get('home', None)
+            dev_home = lgw_entry.get('dev home', None)
+            entry += '- Home: {}\n'.format(', '.join([x for x in [home, dev_home] if x]))

            # state mandatory on our side
            entry += '- State: \n'

            # platform, if existing
            if 'platform' in lgw_entry:
-                entry += 'Platform: {}\n'.format(', '.join(lgw_entry['platform']))
+                entry += '- Platform: {}\n'.format(', '.join(lgw_entry['platform']))

            # keywords (genre) (also mandatory)
            keywords = lgw_entry.get('genre', [])
@ -182,7 +187,7 @@ if __name__ == "__main__":
                entry += '- Keywords: {}\n'.format(', '.join(keywords))

            # code repository (mandatory but not scraped from lgw)
-            entry += '- Code repository: \n'
+            entry += '- Code repository: {}\n'.format(lgw_entry.get('repo', ''))

            # code language, mandatory on our side
            languages = lgw_entry.get('code language', [])
@ -191,6 +196,7 @@ if __name__ == "__main__":

            # code license, mandatory on our side
            licenses = lgw_entry.get('code license', [])
+            licenses = [licenses_map[x] if x in licenses_map else x for x in licenses]
            licenses.sort(key=str.casefold)
            entry += '- Code license: {}\n'.format(', '.join(licenses))

@ -203,7 +209,15 @@ if __name__ == "__main__":

            # assets licenses (only if existing)
            if 'assets license' in lgw_entry:
-                entry += '- Assets license: {}\n'.format(', '.join(lgw_entry['assets license']))
+                licenses = lgw_entry.get('assets license', [])
+                licenses = [licenses_map[x] if x in licenses_map else x for x in licenses]
+                licenses.sort(key=str.casefold)
+                entry += '- Assets license: {}\n'.format(', '.join(licenses))
+
+            # external links
+            ext_links = lgw_entry['external links']
+            if ext_links:
+                entry += '\nLinks: {}\n'.format(', '.join(['{}: {}'.format(x[1], x[0]) for x in ext_links]))

            # free text
            if 'developer' in lgw_entry:
@ -215,5 +229,5 @@ if __name__ == "__main__":
            entry += '\n## Building\n'

            # finally write to file
-            # utils.write_text(target_file, entry)
+            utils.write_text(target_file, entry)
            newly_created_entries += 1
--- a/tools/rejected.txt
+++ b/tools/rejected.txt
@ -17,7 +17,13 @@ Barbie Seahorse Adventures (http://www.imitationpickles.org/barbie/, https://pyw
 Begin2 (https://sourceforge.net/projects/begin2/, https://svn.code.sf.net/p/begin2/code/): No source available
 Berserk (https://github.com/EgorOrachyov/Berserk): No license information found
 biju-game (https://github.com/fdfragoso/biju-game, http://crocidb.com/labs/biju-game/, https://github.com/CrociDB/biju-game): Very early development, No license added
+Brickshooter (http://bilious.homelinux.org/~paxed/brickshooter/): Homepage offline, sources not archived on the web archive, version very low 0.0.4 (might be too early)
+Bweakfwu (https://github.com/plaimi/bweakfwu, https://secure.plaimi.net/games/bweakfwu): Very early development stage, abandoned
 Camelia Girls (https://web.archive.org/web/20140523011334/http://pigux.com:80/cameliagirls/): Sources not available
+Cheese Boys (http://sourceforge.net/projects/cheeseboys/, http://cheeseboys.keul.it/, https://github.com/keul/Cheese-Boys, http://www.pygame.org/project-Cheese+Boys-692-.html): Very early development, Abandoned, License unclear
+Clippers (http://sourceforge.net/projects/clippers/): No license attached to source code (will contact)
+Codewars (http://sourceforge.net/projects/codewars): Very early development phase (especially client)
+CRAFT: The Vicious Vikings (http://web.archive.org/web/20030402062302/borneo.gmd.de/AS/janus/craft/): Source code not available anymore
 Crystal Picnic (https://github.com/Cloudxtreme/crystal-picnic, https://nooskewl.ca/crystal-picnic/): No license of their source code found (or is it zlib?, check again), is commercialized
 Cycle (https://sourceforge.net/projects/meegocycle/): Proprietary license (freeware, source available)
 dart-rpg (https://github.com/brad811/dart-rpg): No license information found