additions from backlog

tools/archives.json

@@ -247,6 +247,7 @@
   "https://github.com/TheAssemblyArmada/Thyme.git",
   "https://github.com/TimPietrusky/asdf.git",
   "https://github.com/TobiasBielefeld/Simple-Solitaire.git",
+  "https://github.com/Try/OpenGothic.git",
   "https://github.com/TurningWheel/Barony.git",
   "https://github.com/Tuxemon/Tuxemon.git",
   "https://github.com/UberGames/rpgxEF.git",
@@ -270,6 +271,7 @@
   "https://github.com/Zetawar/zetawar.git",
   "https://github.com/Zillode/Slot-Racers.git",
   "https://github.com/Zukero/andors-trail.git",
+  "https://github.com/a-nikolaev/curseofwar.git",
   "https://github.com/abhinandanramesh/Star-Wars-III.git",
   "https://github.com/aburch/simutrans.git",
   "https://github.com/acmepjz/meandmyshadow.git",
@@ -299,6 +301,7 @@
   "https://github.com/arescentral/antares.git",
   "https://github.com/arx/ArxLibertatis.git",
   "https://github.com/atrinik/atrinik.git",
+  "https://github.com/b3dgs/lionengine.git",
   "https://github.com/b3dgs/tyrian-remake.git",
   "https://github.com/b3dgs/warcraft-remake.git",
   "https://github.com/basicallydan/skifree.js.git",
@@ -375,6 +378,7 @@
   "https://github.com/endless-sky/endless-sky.git",
   "https://github.com/englercj/lttp-phaser.git",
   "https://github.com/enigma-dev/enigma-dev.git",
+  "https://github.com/estevaofon/angry-birds-python.git",
   "https://github.com/etlegacy/etlegacy.git",
   "https://github.com/exult/exult.git",
   "https://github.com/ezQuake/ezquake-source.git",
@@ -524,6 +528,7 @@
   "https://github.com/omf2097/openomf.git",
   "https://github.com/onaluf/RacerJS.git",
   "https://github.com/ooxi/violetland.git",
+  "https://github.com/openMSX/openMSX.git",
   "https://github.com/openclonk/openclonk.git",
   "https://github.com/openfl/openfl.git",
   "https://github.com/openlegend/core-rules.git",
@@ -578,6 +583,7 @@
   "https://github.com/rofl0r/openDOW.git",
   "https://github.com/rohit-n/Clonepoint.git",
   "https://github.com/rotators/play-fonline-data.git",
+  "https://github.com/rwengine/openrw.git",
   "https://github.com/rystills/rock-raiders-remake.git",
   "https://github.com/sabetts/bratwurst.git",
   "https://github.com/sago007/annchienta.git",

tools/backlog.txt

@@ -3,7 +3,6 @@ DOOM engine (https://github.com/id-Software/DOOM)
 endgame singularity recursive clone (bei anderen auch)
 fosslinux.com
 github - dwarfcorp
-github.com/Attnam/ivan
 gitlab.com/librebob/athenaeum
 Hedgewars
 heXon
@@ -16,20 +15,17 @@ http://icculus.org/avp/
 http://icculus.org/bitstream/
 http://icculus.org/blackshades/
 http://icculus.org/d2x/
-http://icculus.org/duke3d/
 http://icculus.org/freespace2/
 http://icculus.org/freyja/
 http://icculus.org/hge-unix/ http://hge.relishgames.com/
 http://icculus.org/jugglemaster/
 http://icculus.org/jumpnbump/
 http://icculus.org/pyddr/
-http://icculus.org/rott/
 http://icculus.org/tennix/
 http://icculus.org/toby/
 http://libsdl-android.sourceforge.net/
 http://linball.sourceforge.net/
 http://lukaszjakowski.pl/ (games apart from uMario)
-http://opentomb.github.io/
 http://psyco.sourceforge.net/
 http://pyopengl.sourceforge.net/
 http://retrospec.sgn.net
@@ -39,7 +35,6 @@ http://vamos.sourceforge.net/
 http://www.celephais.net/fitzquake/
 http://www.codeskulptor.org/
 http://www.fifengine.net/
-http://www.moddb.com/mods/goldeneye-source
 http://www.myandroidonline.com/category/games/
 http://www.newbreedsoftware.com/s
 http://www.plasmapong.com/plasma-pong/plasma-pong/
@@ -48,7 +43,6 @@ https://appimage.org/
 https://blends.debian.org/games/tasks/
 https://carrot.soulweaver.fi/#links
 https://cocainediesel.fun/
-https://duion.com/games/uebergame/main
 https://empiredirectory.net/
 https://empiredirectory.net/index.php/downloads/viewdownload/6-server-software/13-empire-server
 https://en.wikipedia.org/w/index.php?title=Etterna_(Game)&action=edit&redlink=1
@@ -68,8 +62,8 @@ https://en.wikipedia.org/wiki/GNOME_Games_Collection
 https://en.wikipedia.org/wiki/GNU_Chess
 https://en.wikipedia.org/wiki/GNU_Go
 https://en.wikipedia.org/wiki/List_of_commercial_video_games_with_available_source_code
-https://en.wikipedia.org/wiki/MUD#Spread
-https://en.wikipedia.org/wiki/MUD_client
+https://en.wikipedia.org/wiki/MUD#Spread (all there)
+https://en.wikipedia.org/wiki/MUD_client (all there)
 https://en.wikipedia.org/wiki/No_Gravity_(video_game)
 https://en.wikipedia.org/wiki/One_Hour_One_Life
 https://en.wikipedia.org/wiki/Passage_(video_game)
@@ -88,14 +82,11 @@ https://fedoraproject.org/wiki/SIGs/Games#List_of_games_we_will_NOT_package
 https://flathub.org/home (use it for Linux packaging) / https://flathub.org/apps/category/Game
 https://freegamer.blogspot.com/2015/11/top-3-open-source-pinball-games.html
 https://futurepinball.com/
-https://github.com/a-nikolaev/curseofwar
 https://github.com/afritz1/OpenTESArena
 https://github.com/ajweeks/FlexEngine
 https://github.com/amerkoleci/alimer
 https://github.com/amerkoleci/vortice
-https://github.com/AmrikSadhra/OpenNFS
 https://github.com/amroibrahim/DIYDoom
-https://github.com/angband/angband
 https://github.com/Anthonymcqueen21/Pygame---Alien-Invasion
 https://github.com/ao-libre/ao-cliente
 https://github.com/apsillers/lords-of-the-fey
@@ -103,12 +94,8 @@ https://github.com/apsillers/Taggem
 https://github.com/arianne/stendhal
 https://github.com/ArmageddonGames/ZeldaClassic
 https://github.com/assaultcube (https://sourceforge.net/projects/actiongame/)
-https://github.com/AxioDL/urde
-https://github.com/b3dgs/lionengine
-https://github.com/b3dgs/warcraft-remake
 https://github.com/bitcraft/PyTMX
 https://github.com/BobbyBao/SharpGame
-https://github.com/bvschaik/julius
 https://github.com/ByoxCode/DrawView
 https://github.com/Calinou/libre-pinball
 https://github.com/CatacombGames/
@@ -127,7 +114,6 @@ https://github.com/CompletelyFairGames/dwarfcorp
 https://github.com/Cortrah/SpaceOperaDesign, https://github.com/Cortrah/SpaceOperaRuby/blob/master/design/turnstyles.md
 https://github.com/cping/LGame
 https://github.com/cubei/FlappyCow
-https://github.com/cxong/cdogs-sdl
 https://github.com/cyberegoorg/cetech
 https://github.com/DaemonEngine/Daemon
 https://github.com/Dariasteam/TowerJumper
@@ -140,47 +126,35 @@ https://github.com/ErikLetson/torso-ninja
 https://github.com/ErikLetson/torso-ninja-2
 https://github.com/Ershany/Arcane-Engine
 https://github.com/Esenthel/EsenthelEngine
-https://github.com/estevaofon/angry-birds-python
 https://github.com/etternagame/etterna
 https://github.com/falltergeist/falltergeist
 https://github.com/flathub (all repositories which are games there)
-https://github.com/francot514/FreeSims
 https://github.com/freeboardgame/FreeBoardGame.org
 https://github.com/FreeCol/freecol
-https://github.com/FreezingMoon/AncientBeast
 https://github.com/gamearians
 https://github.com/GentenStudios/quartz-engine
-https://github.com/GlPortal/glPortal
 https://github.com/GunshipPenguin/open_flood
 https://github.com/hedgewars/hw
 https://github.com/hparcells/cards-against-humanity
 https://github.com/IceReaper/DesktopAdventuresToolkit
-https://github.com/Interkarma/daggerfall-unity
-https://github.com/Interrupt/delverengine
 https://github.com/iortcw/iortcw
 https://github.com/jcarolus/android-chess
-https://github.com/juzzlin/DustRacing2D
 https://github.com/lexica/lexica
 https://github.com/libretro/libretro-prboom
 https://github.com/LIKO-12/LIKO-12
 https://github.com/MadDeCoDeR/Classic-RBDOOM-3-BFG
-https://github.com/magefree/mage
 https://github.com/MaKiPL/OpenVIII
 https://github.com/marukrap/ProceduralMapGenerator
 https://github.com/miki151/keeperrl
 https://github.com/MovingBlocks/DestinationSol
 https://github.com/nicolodavis/boardgame.io
 https://github.com/notapixelstudio/superstarfighter
-https://github.com/Novum/vkQuake
 https://github.com/NYRDS/remixed-dungeon
 https://github.com/openblack/openblack
 https://github.com/OpenBricks/openbricks
-https://github.com/OpenDungeons/OpenDungeons
 https://github.com/openitg/openitg
 https://github.com/OpenMandrivaAssociation
-https://github.com/openMSX/openMSX
 https://github.com/opensourcedesign
-https://github.com/opentomb/OpenTomb
 https://github.com/OpMonTeam/OpMon
 https://github.com/OSSGames
 https://github.com/OSSGames/GAME-SDL-ACTION-Blobwars_Metal_Blob_Solid
@@ -190,59 +164,38 @@ https://github.com/pelya/commandergenius
 https://github.com/pld-linux
 https://github.com/ptitSeb/gl4es
 https://github.com/raysan5/rfxgen
-https://github.com/REGoth-project/REGoth-bs
-https://github.com/residualvm/residualvm
 https://github.com/romlok/godot-gdhexgrid
 https://github.com/RonenNess/GeonBit.UI
 https://github.com/RPG-Paper-Maker/RPG-Paper-Maker
 https://github.com/rpgboss/rpgboss
-https://github.com/rwengine/openrw
 https://github.com/saniv/free-game-art/blob/master/foss-copyright-infringement-records.md
 https://github.com/search?p=1&q=sunrider&type=Repositories, sunrider
-https://github.com/shinyquagsire23/DesktopAdventures
-https://github.com/sirjuddington/SLADE
-https://github.com/smcameron/space-nerds-in-space
 https://github.com/SneakyTactician/MagicalLife
-https://github.com/sourcehold/Sourcehold
 https://github.com/sphair/ClanLib
 https://github.com/sromku/memory-game
 https://github.com/Stabyourself/mari0
 https://github.com/stephenjsweeney/blobwarsAttrition
-https://github.com/STJr/SRB2
 https://github.com/suijingfeng/vkOpenArena
-https://github.com/SuperTux/supertux
-https://github.com/taisei-project/taisei
 https://github.com/TalonBraveInfo/OpenHoW
 https://github.com/team-eternity/eternity
 https://github.com/the-insulines
-https://github.com/The-Powder-Toy/The-Powder-Toy
 https://github.com/theaigames
-https://github.com/TheAssemblyArmada/Chronoshift
-https://github.com/TheAssemblyArmada/Thyme
 https://github.com/tonihele/OpenKeeper
 https://github.com/tpcstld/2048
-https://github.com/Try/OpenGothic
 https://github.com/unknown-horizons/godot-port
-https://github.com/ValyriaTear/ValyriaTear
 https://github.com/vgc/vgc
 https://github.com/vilbeyli/VQEngine
 https://github.com/Vocaluxe/Vocaluxe
 https://github.com/Wargus/stratagus
 https://github.com/wesnoth/haldric
-https://github.com/xLAva/JediOutcastLinux
-https://github.com/XProger/OpenLara
 https://github.com/xyzz/openmw-android
 https://github.com/yairm210/UnCiv
 https://github.com/yiotro/Antiyoy
-https://github.com/yquake2/yquake2
 https://gitlab.com/LibreGames
 https://gitlab.com/vgstation/vgstation-old (and vgstation in general)
 https://keeperrl.com/
 https://lgdb.org/game/wolfpack-empire
 https://libregamewiki.org/Libregamewiki:Suggested_games#Likely_sources_for_more_free_games
-https://neverball.org/
-https://nukeykt.retrohost.net/
-https://openage.sft.mx/
 https://osdn.net/softwaremap/trove_list.php?form_cat=80
 https://packages.debian.org/sid/games/etw
 https://pixeldoctrine.com/index.html (more than just slime volleyball)
@@ -268,10 +221,7 @@ https://sourceforge.net/projects/palomino-sim/ (http://www.jimbrooks.org/archive
 https://sourceforge.net/projects/pinball/
 https://sourceforge.net/projects/pixellight/
 https://sourceforge.net/projects/realmforge/
-https://sourceforge.net/projects/uhexen2/
-https://sourceforge.net/projects/vpinball/
 https://sourceforge.net/projects/webchess/
-https://sourceforge.net/projects/zaz/
 https://stabyourself.net/ (all projects there)
 https://web.archive.org/web/20150526225753/http://sourceforge.net/projects/mameosx/
 https://web.archive.org/web/20150527150705/http://sourceforge.net/p/forge/community-docs/SourceForge%20Games/ (schon erledigt)

tools/maintenance.py

@@ -109,8 +109,8 @@ def check_validity_external_links():
     print("check external links (can take a while)")
 
     # regex for finding urls (can be in <> or in ]() or after a whitespace
-    #regex = re.compile(r"[\s\n]<(http.+?)>|\]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n\)]")
-    regex = re.compile(r"[\s\n<(](http://.*?)[\s\n>)]")
+    regex = re.compile(r"[\s\n]<(http.+?)>|\]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n\)]")
+    # regex = re.compile(r"[\s\n<(](http://.*?)[\s\n>)]")
 
     # count
     number_checked_links = 0
@@ -178,6 +178,8 @@ def fix_entries():
     Fixes the keywords, code dependencies, build systems, .. entries, mostly by automatically sorting them.
     """
 
+    keyword_synonyms = {'RTS': ('real time', 'strategy'), 'realtime': 'real time'}
+
     # TODO also sort other fields, only read once and then do all, move to separate file
 
     print('fix entries')
@@ -398,6 +400,8 @@ def update_statistics(infos):
     for info in infos:
         if field in info:
             keywords.extend(info[field])
+    # ignore those starting with "inspired by"
+    keywords = [x for x in keywords if not x.startswith('inspired by ')]
 
     unique_keywords = set(keywords)
     unique_keywords = [(l, keywords.count(l) / len(keywords)) for l in unique_keywords]
@@ -743,6 +747,32 @@ def sort_text_file(file, name):
     text = '\n'.join(text)
     write_text(file, text)
 
+def strip_url(url):
+    for prefix in ('http://', 'https://'):
+        if url.startswith(prefix):
+            url = url[len(prefix):]
+    for suffix in ('/', '.git'):
+        if url.endswith(suffix):
+            url = url[:-len(suffix)]
+    return url
+
+def clean_backlog(stripped_game_urls):
+
+    # read backlog and split
+    file = os.path.join(root_path, 'tools', 'backlog.txt')
+    text = read_text(file)
+    text = text.split('\n')
+
+    # remove those that are in stripped_game_urls
+    text = [x for x in text if strip_url(x) not in stripped_game_urls]
+
+    # remove duplicates and sort
+    text = sorted(list(set(text)), key=str.casefold)
+    print('backlog contains {} items'.format(len(text)))
+
+    # join and save again
+    text = '\n'.join(text)
+    write_text(file, text)
 
 if __name__ == "__main__":
 
@@ -750,6 +780,12 @@ if __name__ == "__main__":
     root_path  = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir))
     games_path = os.path.join(root_path, 'games')
 
+    # backlog
+    game_urls = extract_links(games_path)
+    stripped_game_urls = [strip_url(x) for x in game_urls]
+    clean_backlog(stripped_game_urls)
+
+
     # check for unfilled template lines
     check_template_leftovers()
 
@@ -778,5 +814,5 @@ if __name__ == "__main__":
     # check_validity_external_links()
 
     # sort backlog and rejected
-    sort_text_file(os.path.join(root_path, 'tools', 'backlog.txt'), 'backlog')
+    # sort_text_file(os.path.join(root_path, 'tools', 'backlog.txt'), 'backlog')
     sort_text_file(os.path.join(root_path, 'tools', 'rejected.txt'), 'rejected games list')

tools/utils/osg.py

@@ -223,4 +223,31 @@ def assemble_infos(games_path):
         # add to list
         infos.append(info)
 
-    return infos
+    return infos
+
+def extract_links(games_path):
+    """
+    Parses all entries and extracts http(s) links from them
+    """
+
+    # regex for finding urls (can be in <> or in ]() or after a whitespace
+    regex = re.compile(r"[\s\n]<(http.+?)>|\]\((http.+?)\)|[\s\n](http[^\s\n,]+?)[\s\n,]")
+
+    # iterate over all entries
+    urls = set()
+    for _, _, content in entry_iterator(games_path):
+
+        # apply regex
+        matches = regex.findall(content)
+
+        # for each match
+        for match in matches:
+
+            # for each possible clause
+            for url in match:
+
+                # if there was something (and not a sourceforge git url)
+                if url:
+                    urls.add(url)
+    urls = sorted(list(urls), key=str.casefold)
+    return urls