synchronization script for awesome lists, git submodules updated

code/archive_detect_submodules.py

@@ -5,9 +5,9 @@ repositories to be checked out. Works on bare repositories.
 
 import json
 import re
-
-from utils.utils import *
-from utils.archive import *
+import os
+import urllib
+from utils import constants as c, utils as u, archive as a
 
 if __name__ == '__main__':
 
@@ -15,23 +15,25 @@ if __name__ == '__main__':
 
     # get this folder
     root_folder = os.path.realpath(os.path.dirname(__file__))
-    archive_folder = os.path.join(root_folder, 'archive')
+    archive_folder = c.get_config('archive-folder')
+    base_folder = os.path.join(archive_folder, 'git')
 
     # read archives.json
-    text = read_text(os.path.join(root_folder, 'archives.json'))
+    text = u.read_text(os.path.join(root_folder, 'archives.json'))
     archives = json.loads(text)
 
     # loop over all git archives
     submodules = []
     for repo in archives['git']:
-        git_folder = git_folder_name(repo)
+        print('process {}'.format(repo))
+        git_folder = a.git_folder_name(repo)
         folder = os.path.join(archive_folder, 'git', git_folder)
         if not os.path.isdir(folder):
             print('Warning: folder {} does not exist'.format(git_folder))
             continue
         os.chdir(folder)
         try:
-            content = subprocess_run(['git', 'show', 'HEAD:.gitmodules'], False)
+            content = u.subprocess_run(['git', 'show', 'HEAD:.gitmodules'], False)
         except:
             continue
         matches = regex_submodules.findall(content)
@@ -55,4 +57,4 @@ if __name__ == '__main__':
 
     # store them
     print('found {} submodules'.format(len(submodules)))
-    write_text(os.path.join(root_folder, 'archives.git-submodules.json'), json.dumps(submodules, indent=1))
+    u.write_text(os.path.join(root_folder, 'archives.git-submodules.json'), json.dumps(submodules, indent=1))

code/archive_update.py

@@ -7,16 +7,15 @@ Uses 'git clone --mirror' to set up the git locally.
 
 Warning: This may take a long time on the first run and may need a lot of storage space!
 
-TODO are really all existing branches cloned and pulled? (see https://stackoverflow.com/questions/67699/how-to-clone-all-remote-branches-in-git)
-TODO Sourceforge git clone may not work all the time (restarting the script sometimes helps..)
-
 Note: May need to set http.postBuffer (https://stackoverflow.com/questions/17683295/git-bash-error-rpc-failed-result-18-htp-code-200b-1kib-s)
 
-
 For repositories
 see https://serverfault.com/questions/544156/git-clone-fail-instead-of-prompting-for-credentials
 """
 
+# TODO are really all existing branches cloned and pulled? (see https://stackoverflow.com/questions/67699/how-to-clone-all-remote-branches-in-git)
+# TODO Sourceforge git clone may not work all the time (restarting the script sometimes helps..)
+
 import json
 
 from utils.utils import *
@@ -174,7 +173,7 @@ if __name__ == '__main__':
     archives_git_submodules = json.loads(text)
 
     # run update on submodules
-    # run_update('git', archives_git_submodules, 'git-submodules')
+    run_update('git', archives_git_submodules, 'git-submodules')
 
     # update
     for type in archives:

code/archives.git-submodules.json

@@ -1,40 +1,56 @@
 [
  "git://git.guelker.eu/pod-cpp.git",
- "git://git.guelker.eu/tinyclipboard.git",
  "git://git.thousandparsec.net/git/schemepy.git",
- "http://luajit.org/git/luajit-2.0.git",
+ "https://bitbucket.org/cafu/textures.git",
  "https://bitbucket.org/ecwolf/sdl.git",
  "https://bitbucket.org/ecwolf/sdl_mixer-for-ecwolf.git",
  "https://bitbucket.org/ecwolf/sdl_net.git",
  "https://bitbucket.org/nyan_developer/nya-engine.git",
- "https://boringssl.googlesource.com/boringssl",
  "https://chromium.googlesource.com/breakpad/breakpad",
- "https://github.com/Aleph-One-Marathon/data-marathon-2.git",
- "https://github.com/Aleph-One-Marathon/data-marathon-infinity.git",
- "https://github.com/Aleph-One-Marathon/data-marathon.git",
+ "https://github.com/AliveTeam/fluidsynth-lite.git",
  "https://github.com/AmrikSadhra/g3log.git",
  "https://github.com/AndresTraks/BulletSharpPInvoke.git",
+ "https://github.com/AtomicGameEngine/AtomicExamples.git",
+ "https://github.com/AtomicGameEngine/CEF3Binaries.git",
  "https://github.com/BalazsJako/ImGuiColorTextEdit.git",
- "https://github.com/BrianGladman/sha.git",
+ "https://github.com/BlackMATov/ecs.hpp.git",
+ "https://github.com/BlackMATov/enum.hpp.git",
+ "https://github.com/BlackMATov/flat.hpp.git",
+ "https://github.com/BlackMATov/pnpoly.h.git",
+ "https://github.com/BlackMATov/promise.hpp.git",
  "https://github.com/CallumDev/FontConfigSharp.git",
  "https://github.com/CallumDev/lidgren-network-gen3.git",
+ "https://github.com/ColinDuquesnoy/QDarkStyleSheet.git",
+ "https://github.com/DLTcollab/sse2neon.git",
  "https://github.com/DaemonEngine/CBSE-Toolchain.git",
  "https://github.com/DaemonEngine/breakpad.git",
  "https://github.com/DaemonEngine/crunch.git",
  "https://github.com/DaemonEngine/recastnavigation.git",
+ "https://github.com/Dav1dde/glad.git",
+ "https://github.com/DethRaid/rex.git",
+ "https://github.com/DethRaid/vcpkg.git",
  "https://github.com/DusteDdk/list.git",
  "https://github.com/Extrawurst/cimgui.git",
  "https://github.com/FIX94/fixNES.git",
  "https://github.com/FNA-XNA/FNA.git",
  "https://github.com/Facepunch/Facepunch.Steamworks.git",
+ "https://github.com/GPUOpen-Effects/FidelityFX-SPD.git",
+ "https://github.com/GPUOpen-LibrariesAndSDKs/AGS_SDK.git",
  "https://github.com/GTA-ASM/MirrorLite.git",
+ "https://github.com/GTA-ASM/UnityStandardAssets.git",
  "https://github.com/GameFoundry/bsf.git",
  "https://github.com/GentenStudios/bgfx.cmake.git",
- "https://github.com/GlPortal/RadixEngine.git",
+ "https://github.com/Giperion/X-Ray-Oxygen-Xbox.git",
+ "https://github.com/GlPortal/RadixEntity.git",
+ "https://github.com/GlPortal/cpp-linenoise.git",
  "https://github.com/GlPortal/documentation.git",
  "https://github.com/GlPortal/glportal-data.git",
- "https://github.com/Grumbel/uitest.git",
+ "https://github.com/GlPortal/serine.git",
+ "https://github.com/GlPortal/vhacd-lib.git",
+ "https://github.com/Grumbel/shared-modules.git",
+ "https://github.com/HowardHinnant/date.git",
  "https://github.com/Jakz/openmom-editor.git",
+ "https://github.com/JochenKalmbach/StackWalker.git",
  "https://github.com/JonnyH/glm.git",
  "https://github.com/JonnyH/libsmacker.git",
  "https://github.com/JonnyH/miniz.git",
@@ -42,23 +58,41 @@
  "https://github.com/JuliaStrings/utf8proc.git",
  "https://github.com/KhronosGroup/SPIRV-Headers.git",
  "https://github.com/KhronosGroup/Vulkan-Headers.git",
+ "https://github.com/KhronosGroup/glTF.git",
+ "https://github.com/KhronosGroup/glslang.git",
  "https://github.com/LaurentGomila/SFML.git",
  "https://github.com/LibVNC/libvncserver.git",
  "https://github.com/Librelancer/Collada141.git",
+ "https://github.com/Librelancer/ImGui.NET.git",
+ "https://github.com/Librelancer/lancerdecode.git",
+ "https://github.com/Librelancer/moonsharp.git",
+ "https://github.com/Librelancer/thorncompiler.git",
  "https://github.com/Lyndir/Pearl.git",
  "https://github.com/Lyndir/love-lyndir.client.git",
  "https://github.com/MegaGlest/megaglest-data.git",
  "https://github.com/MegaGlest/megaglest-masterserver.git",
  "https://github.com/MegaGlest/mojosetup-fork.git",
- "https://github.com/Microsoft/DirectXTex.git",
+ "https://github.com/Microsoft/vcpkg.git",
  "https://github.com/MonoGame/MonoGame.Dependencies.git",
+ "https://github.com/NVIDIAGameWorks/PhysX.git",
+ "https://github.com/Neargye/magic_enum.git",
  "https://github.com/OpenFodder/data.git",
+ "https://github.com/OpenXRay/BugTrap.git",
+ "https://github.com/OpenXRay/FreeMagic.git",
+ "https://github.com/OpenXRay/GameSpy.git",
+ "https://github.com/OpenXRay/LuaJIT.git",
+ "https://github.com/OpenXRay/OpenAutomate.git",
+ "https://github.com/OpenXRay/libjpeg.git",
+ "https://github.com/OpenXRay/luabind-deboostified.git",
+ "https://github.com/OpenXRay/mimalloc.git",
  "https://github.com/Orphis/boost-cmake.git",
+ "https://github.com/OutpostUniverse/ophd-assets.git",
  "https://github.com/Perlmint/glew-cmake.git",
  "https://github.com/Ponup/api-cpp-client.git",
  "https://github.com/Ponup/engine-desktop.git",
  "https://github.com/REGoth-project/BsZenLib.git",
  "https://github.com/REGoth-project/CAB-Installer-Extractor.git",
+ "https://github.com/Return-To-The-Roots/dev-tools.git",
  "https://github.com/Return-To-The-Roots/languages.git",
  "https://github.com/Return-To-The-Roots/libendian.git",
  "https://github.com/Return-To-The-Roots/liblobby.git",
@@ -66,49 +100,54 @@
  "https://github.com/Return-To-The-Roots/libutil.git",
  "https://github.com/Return-To-The-Roots/mygettext.git",
  "https://github.com/Return-To-The-Roots/s25edit.git",
+ "https://github.com/Return-To-The-Roots/s25maps.git",
  "https://github.com/Return-To-The-Roots/s25update.git",
  "https://github.com/RigsOfRods/content.git",
  "https://github.com/RigsOfRods/ogre-pagedgeometry.git",
  "https://github.com/Robmaister/SharpFont.git",
+ "https://github.com/SRombauts/SQLiteCpp.git",
  "https://github.com/Sebanisu/zzzDeArchive.git",
  "https://github.com/SirCmpwn/fNbt.git",
+ "https://github.com/SoupeauCaillou/sac.git",
  "https://github.com/StbSharp/StbImageSharp.git",
  "https://github.com/StbSharp/StbImageWriteSharp.git",
  "https://github.com/SuperTux/SDL_ttf.git",
  "https://github.com/SuperTux/physfs.git",
  "https://github.com/SuperTux/sexp-cpp.git",
  "https://github.com/SuperTux/tinygettext.git",
- "https://github.com/SuperV1234/SSVLuaWrapper.git",
  "https://github.com/SuperV1234/SSVMenuSystem.git",
  "https://github.com/SuperV1234/SSVStart.git",
  "https://github.com/SuperV1234/SSVUtils.git",
  "https://github.com/SuperV1234/vrm_cmake.git",
  "https://github.com/SuperV1234/vrm_pp.git",
- "https://github.com/TalonBraveInfo/newton-dynamics.git",
  "https://github.com/TalonBraveInfo/platform.git",
  "https://github.com/TartanLlama/optional.git",
  "https://github.com/Tencent/rapidjson.git",
  "https://github.com/TheAssemblyArmada/BaseConfig.git",
- "https://github.com/TheAssemblyArmada/Baseconfig.git",
  "https://github.com/TheAssemblyArmada/CaptainsLog.git",
  "https://github.com/TheAssemblyArmada/SetSail.git",
- "https://github.com/TheAssemblyArmada/gamemath.git",
  "https://github.com/ThePhD/sol2.git",
- "https://github.com/Try/MoltenTempest.git",
+ "https://github.com/Try/Tempest.git",
  "https://github.com/Try/ZenLib.git",
  "https://github.com/TsudaKageyu/minhook.git",
+ "https://github.com/USCiLab/cereal.git",
  "https://github.com/Unvanquished/libRocket.git",
  "https://github.com/UnvanquishedAssets/unvanquished_src.dpkdir.git",
  "https://github.com/ValyriaTear/luabind.git",
  "https://github.com/ValyriaTear/vt-utils.git",
+ "https://github.com/Warzone2100/data-music.git",
+ "https://github.com/Warzone2100/data-texpages.git",
  "https://github.com/Wohlstand/libADLMIDI/.git",
  "https://github.com/aap/geniedoc.git",
  "https://github.com/accumulators.git",
+ "https://github.com/ajweeks/bullet3.git",
  "https://github.com/ajweeks/glm.git",
  "https://github.com/albertodemichelis/squirrel.git",
  "https://github.com/alecthomas/entityx.git",
+ "https://github.com/alexgdi/lzo.git",
  "https://github.com/algorithm.git",
  "https://github.com/align.git",
+ "https://github.com/andreakarasho/MP3Sharp.git",
  "https://github.com/anttisalonen/libcommon.git",
  "https://github.com/antze-k/miso.git",
  "https://github.com/anura-engine/imgui.git",
@@ -116,12 +155,16 @@
  "https://github.com/arescentral/antares-data.git",
  "https://github.com/arescentral/antares-test-data.git",
  "https://github.com/arescentral/procyon.git",
+ "https://github.com/aroldanju/caengine.git",
+ "https://github.com/aroldanju/ini-reader.git",
+ "https://github.com/aroldanju/res-packager.git",
  "https://github.com/array.git",
  "https://github.com/arsenm/sanitizers-cmake.git",
  "https://github.com/aseprite/freetype2.git",
  "https://github.com/asio.git",
  "https://github.com/assert.git",
  "https://github.com/assign.git",
+ "https://github.com/assimp/assimp.git",
  "https://github.com/atomic.git",
  "https://github.com/atrinik/atrinik-sound.git",
  "https://github.com/atrinik/resources.git",
@@ -132,22 +175,28 @@
  "https://github.com/beast.git",
  "https://github.com/bimap.git",
  "https://github.com/bind.git",
+ "https://github.com/biojppm/rapidyaml.git",
  "https://github.com/bjorn/tiled.git",
+ "https://github.com/blackmatov/defer.hpp.git",
+ "https://github.com/bombela/backward-cpp.git",
  "https://github.com/boost_install.git",
  "https://github.com/boostbook.git",
  "https://github.com/boostdep.git",
  "https://github.com/build.git",
  "https://github.com/bulletphysics/bullet3.git",
  "https://github.com/butter-cat-games/cimguilibs.git",
- "https://github.com/c42f/tinyformat.git",
  "https://github.com/callable_traits.git",
+ "https://github.com/cameron314/readerwriterqueue.git",
  "https://github.com/canta-media.git",
- "https://github.com/castano/nvidia-texture-tools.git",
+ "https://github.com/carstenfuchs/glfw.git",
+ "https://github.com/catchorg/Catch2.git",
+ "https://github.com/cflavio/yracing.git",
  "https://github.com/cflavio/yyagl.git",
  "https://github.com/check_build.git",
  "https://github.com/chocolate-doom/quickcheck.git",
  "https://github.com/chrono.git",
  "https://github.com/circular_buffer.git",
+ "https://github.com/cmake.git",
  "https://github.com/cocos2d/bindings-generator.git",
  "https://github.com/cocos2d/cocos2d-console.git",
  "https://github.com/cocos2d/cocos2d-html5.git",
@@ -179,32 +228,42 @@
  "https://github.com/crawl/crawl-zlib.git",
  "https://github.com/crc.git",
  "https://github.com/cxong/tinydir.git",
+ "https://github.com/data-marathon-2.git",
+ "https://github.com/data-marathon-infinity.git",
+ "https://github.com/data-marathon.git",
  "https://github.com/date_time.git",
  "https://github.com/dbry/adpcm-xq.git",
  "https://github.com/degenerated1123/ZenLib.git",
  "https://github.com/degenerated1123/bgfx-cmake.git",
+ "https://github.com/describe.git",
  "https://github.com/detail.git",
+ "https://github.com/discord/discord-rpc.git",
  "https://github.com/discordapp/discord-rpc.git",
- "https://github.com/disjoint_sets.git",
  "https://github.com/dll.git",
  "https://github.com/docca.git",
  "https://github.com/dotfloat/fluidsynth-lite.git",
- "https://github.com/dr-soft/mini_al.git",
+ "https://github.com/dr-soft/miniaudio.git",
  "https://github.com/droidmonkey/python-cmake-buildsystem.git",
  "https://github.com/dumganhar/ccs-res.git",
  "https://github.com/dynamic_bitset.git",
  "https://github.com/egoboo/egoboo-assets.git",
  "https://github.com/egoboo/egoboo-external.git",
  "https://github.com/egoboo/idlib-game-engine.git",
+ "https://github.com/elonafoobar/windows_deps.git",
  "https://github.com/endian.git",
- "https://github.com/erikd/libsamplerate.git",
  "https://github.com/etlegacy/etlegacy-libs.git",
  "https://github.com/exception.git",
+ "https://github.com/ezEngine/content.git",
+ "https://github.com/ezEngine/precompiled-tools.git",
+ "https://github.com/ezEngine/thirdparty-fmod.git",
+ "https://github.com/ezEngine/thirdparty-physx.git",
+ "https://github.com/facebook/yoga.git",
  "https://github.com/fiber.git",
  "https://github.com/filesystem.git",
  "https://github.com/flathub/shared-modules.git",
  "https://github.com/flyweight.git",
  "https://github.com/fmtlib/fmt.git",
+ "https://github.com/foonathan/type_safe.git",
  "https://github.com/foreach.git",
  "https://github.com/format.git",
  "https://github.com/frabert/libdmusic.git",
@@ -215,6 +274,7 @@
  "https://github.com/functional.git",
  "https://github.com/fusion.git",
  "https://github.com/fuzzylite/fuzzylite.git",
+ "https://github.com/g-truc/gli.git",
  "https://github.com/g-truc/glm.git",
  "https://github.com/gabomdq/SDL_GameControllerDB.git",
  "https://github.com/geometry.git",
@@ -223,6 +283,9 @@
  "https://github.com/glfw/glfw.git",
  "https://github.com/gliptic/tl.git",
  "https://github.com/google/googletest.git",
+ "https://github.com/google/re2.git",
+ "https://github.com/google/wycheproof.git",
+ "https://github.com/gost-engine/engine.git",
  "https://github.com/graph.git",
  "https://github.com/graph_parallel.git",
  "https://github.com/grit-engine/grit-bullet.git",
@@ -256,12 +319,17 @@
  "https://github.com/ivucica/glict.git",
  "https://github.com/ivucica/rules_libsdl12.git",
  "https://github.com/ivucica/rules_tibia.git",
+ "https://github.com/jaihysc/libnoise.git",
  "https://github.com/jarikomppa/soloud.git",
  "https://github.com/jarro2783/cxxopts.git",
  "https://github.com/jazztickets/ae.git",
  "https://github.com/jazztickets/cmake.git",
+ "https://github.com/jfaudiolib.git",
+ "https://github.com/jfbuild.git",
+ "https://github.com/jfmact.git",
  "https://github.com/jhaynie/iphonesim.git",
  "https://github.com/jonasmr/microprofile.git",
+ "https://github.com/json.git",
  "https://github.com/kaadmy/pixture.git",
  "https://github.com/kcat/openal-soft.git",
  "https://github.com/kebby/assimp-net.git",
@@ -269,7 +337,9 @@
  "https://github.com/kuba--/zip.git",
  "https://github.com/lairworks/nas2d-core.git",
  "https://github.com/lambda.git",
+ "https://github.com/lambda2.git",
  "https://github.com/laradock/laradock.git",
+ "https://github.com/leaf.git",
  "https://github.com/leethomason/tinyxml2.git",
  "https://github.com/les-sosna/ios-cmake.git",
  "https://github.com/lexical_cast.git",
@@ -277,46 +347,49 @@
  "https://github.com/libretro/libretro-common.git",
  "https://github.com/libtom/libtommath.git",
  "https://github.com/libvorbis-1.3.3.git",
- "https://github.com/lispparser/sexp-cpp.git",
+ "https://github.com/lilybeevee/bab-be-u.wiki.git",
  "https://github.com/litre.git",
  "https://github.com/local_function.git",
  "https://github.com/locale.git",
  "https://github.com/lockfree.git",
  "https://github.com/log.git",
  "https://github.com/logic.git",
- "https://github.com/logmich/logmich.git",
  "https://github.com/lubomyr/uae4all2.git",
  "https://github.com/lubomyr/vice-2.4.git",
  "https://github.com/lvandeve/lodepng.git",
  "https://github.com/mat007/turtle.git",
  "https://github.com/math.git",
- "https://github.com/meganz/mingw-std-threads.git",
- "https://github.com/mellinoe/ImGui.NET.git",
  "https://github.com/memononen/fontstash.git",
  "https://github.com/metaparse.git",
+ "https://github.com/microsoft/vcpkg.git",
  "https://github.com/miloyip/rapidjson.git",
  "https://github.com/miniupnp/miniupnp.git",
+ "https://github.com/mlabbe/nativefiledialog.git",
  "https://github.com/mmatyas/libSDL2pp.git",
  "https://github.com/mmatyas/unittest-cpp.git",
+ "https://github.com/more.git",
+ "https://github.com/moritz-wundke/Boost-for-Android.git",
  "https://github.com/move.git",
  "https://github.com/mp11.git",
  "https://github.com/mpi.git",
  "https://github.com/mpl.git",
+ "https://github.com/mpreisler/mingw-bundledlls.git",
  "https://github.com/mruby/mruby.git",
  "https://github.com/msgpack/msgpack-c.git",
  "https://github.com/msm.git",
  "https://github.com/multi_array.git",
  "https://github.com/multi_index.git",
  "https://github.com/multiprecision.git",
+ "https://github.com/nashmuhandes/WidePix.git",
  "https://github.com/nemtrif/utfcpp.git",
  "https://github.com/nielsAD/travis-lazarus.git",
  "https://github.com/nlohmann/json.git",
  "https://github.com/nothings/stb.git",
  "https://github.com/nowide.git",
  "https://github.com/numeric_conversion.git",
- "https://github.com/numpy/numpydoc.git",
+ "https://github.com/nyorain/dlg.git",
  "https://github.com/o11c/attoconf.git",
- "https://github.com/oamldev/oaml.git",
+ "https://github.com/obfusk/bigbang.coffee.git",
  "https://github.com/odeint.git",
  "https://github.com/oolite-binary-resources.git",
  "https://github.com/oolite-linux-dependencies.git",
@@ -326,10 +399,14 @@
  "https://github.com/oolite-windows-dependencies.git",
  "https://github.com/ooxi/CMake-Gettext.git",
  "https://github.com/open-source-parsers/jsoncpp.git",
+ "https://github.com/openblack/bgfx.cmake.git",
  "https://github.com/optional.git",
  "https://github.com/outcome.git",
  "https://github.com/parameter.git",
  "https://github.com/parameter_python.git",
+ "https://github.com/past-due/EmbeddedJSONSignature.git",
+ "https://github.com/past-due/discord-rpc.git",
+ "https://github.com/past-due/launchinfo.git",
  "https://github.com/paulsapps/Detours.git",
  "https://github.com/paulsapps/SUDM.git",
  "https://github.com/paulsapps/TinyXML.git",
@@ -342,11 +419,11 @@
  "https://github.com/paulsapps/sqrat.git",
  "https://github.com/paulsapps/squirrel.git",
  "https://github.com/pelya/BasiliskII-android.git",
- "https://github.com/pelya/Boost-for-Android.git",
  "https://github.com/pelya/Ninslash.git",
  "https://github.com/pelya/OpenTTD-JGR-patchpack.git",
  "https://github.com/pelya/android-keyboard-gadget.git",
  "https://github.com/pelya/android-shmem.git",
+ "https://github.com/pelya/debian-noroot.git",
  "https://github.com/pelya/libiconv-libicu-android.git",
  "https://github.com/pelya/liero-android.git",
  "https://github.com/pelya/openarena-engine.git",
@@ -355,10 +432,10 @@
  "https://github.com/pelya/pulseaudio-android.git",
  "https://github.com/pelya/supertux.git",
  "https://github.com/pelya/teeworlds.git",
- "https://github.com/pelya/xmoto.git",
  "https://github.com/pelya/xserver.git",
- "https://github.com/performous/compact_enc_det.git",
  "https://github.com/petroules/solar-cmake.git",
+ "https://github.com/pfr.git",
+ "https://github.com/philsquared/Catch.git",
  "https://github.com/phoenix.git",
  "https://github.com/poly_collection.git",
  "https://github.com/polygon.git",
@@ -369,13 +446,16 @@
  "https://github.com/process.git",
  "https://github.com/program_options.git",
  "https://github.com/property_map.git",
+ "https://github.com/property_map_parallel.git",
  "https://github.com/property_tree.git",
  "https://github.com/proto.git",
  "https://github.com/ptr_container.git",
+ "https://github.com/pybind/pybind11.git",
  "https://github.com/pyca/cryptography.git",
  "https://github.com/python.git",
  "https://github.com/q-gears/data.git",
  "https://github.com/q-gears/luajit.git",
+ "https://github.com/qbs/qbs.git",
  "https://github.com/qml-box2d/qml-box2d.git",
  "https://github.com/qnighy/ruby-1.8.1.git",
  "https://github.com/qnighy/ruby-1.9.2p0.git",
@@ -387,6 +467,7 @@
  "https://github.com/rational.git",
  "https://github.com/rds1983/StbSharp.git",
  "https://github.com/recastnavigation/recastnavigation.git",
+ "https://github.com/recp/cglm.git",
  "https://github.com/redeclipse/acerspyro.git",
  "https://github.com/redeclipse/actors.git",
  "https://github.com/redeclipse/appleflap.git",
@@ -426,15 +507,17 @@
  "https://github.com/redeclipse/vegetation.git",
  "https://github.com/redeclipse/weapons.git",
  "https://github.com/regex.git",
+ "https://github.com/richgel999/miniz.git",
  "https://github.com/riperiperi/FSOMina.NET.git",
  "https://github.com/riperiperi/FSOMonoGame.git",
  "https://github.com/safe_numerics.git",
  "https://github.com/sakra/cotire.git",
  "https://github.com/sandsmark/genieutils.git",
  "https://github.com/satoren/kaguya.git",
+ "https://github.com/scemino/EnggeFramework.git",
  "https://github.com/schellingb/TinySoundFont.git",
- "https://github.com/scipy/scipy-sphinx-theme.git",
  "https://github.com/scope_exit.git",
+ "https://github.com/scummvm/game-translations.git",
  "https://github.com/serialization.git",
  "https://github.com/sfiera/glfw-gyp.git",
  "https://github.com/sfiera/gmock-gyp.git",
@@ -445,6 +528,7 @@
  "https://github.com/sfiera/libsndfile-gyp.git",
  "https://github.com/sfiera/libzipxx.git",
  "https://github.com/sfiera/zlib-gn.git",
+ "https://github.com/shlusiak/Freebloks-3D.git",
  "https://github.com/signals2.git",
  "https://github.com/singularity/singularity-music-lossless-extended.git",
  "https://github.com/singularity/singularity-music-lossless-original.git",
@@ -452,11 +536,12 @@
  "https://github.com/singularity/singularity-osx.git",
  "https://github.com/singularity/singularity-windows.git",
  "https://github.com/skaslev/gl3w.git",
- "https://github.com/skyjake/assimp.git",
+ "https://github.com/skypjack/entt.git",
  "https://github.com/smart_ptr.git",
  "https://github.com/sort.git",
  "https://github.com/spidermonkey-ff4.git",
  "https://github.com/spirit.git",
+ "https://github.com/spring/AAI.git",
  "https://github.com/spring/CircuitAI.git",
  "https://github.com/spring/HughAI.git",
  "https://github.com/spring/KAIK.git",
@@ -465,15 +550,11 @@
  "https://github.com/spring/SpringMapConvNG.git",
  "https://github.com/spring/pr-downloader.git",
  "https://github.com/spring/pyunitsync.git",
- "https://github.com/spring1944/submodule-core.git",
- "https://github.com/spring1944/submodule-customCommands.git",
- "https://github.com/spring1944/submodule-goals.git",
- "https://github.com/spring1944/submodule-notAchili.git",
- "https://github.com/spring1944/submodule-strongpoints.git",
  "https://github.com/spurious/SDL-mirror.git",
  "https://github.com/stacktrace.git",
  "https://github.com/statechart.git",
  "https://github.com/static_assert.git",
+ "https://github.com/static_string.git",
  "https://github.com/stephank/villain.git",
  "https://github.com/stepmania/ffmpeg.git",
  "https://github.com/stepmania/googletest.git",
@@ -481,14 +562,18 @@
  "https://github.com/stepmania/libtommath.git",
  "https://github.com/stepmania/ogg.git",
  "https://github.com/stepmania/vorbis.git",
+ "https://github.com/stl_interfaces.git",
  "https://github.com/syoyo/tinyobjloader.git",
  "https://github.com/system.git",
  "https://github.com/taisei-project/SDL_GameControllerDB.git",
- "https://github.com/taisei-project/cglm.git",
+ "https://github.com/taisei-project/basis_universal.git",
+ "https://github.com/taisei-project/koishi.git",
+ "https://github.com/taisei-project/python-zipfile-zstd.git",
  "https://github.com/team-cube/open-cube-data.git",
  "https://github.com/teeworlds/teeworlds-maps.git",
  "https://github.com/teeworlds/teeworlds-translation.git",
  "https://github.com/test.git",
+ "https://github.com/thedmd/imgui-node-editor.git",
  "https://github.com/themanaworld/tmw-music.git",
  "https://github.com/themanaworld/tmw-tools.git",
  "https://github.com/thestk/stk.git",
@@ -498,59 +583,89 @@
  "https://github.com/tinygettext/tinygettext.git",
  "https://github.com/tokenizer.git",
  "https://github.com/tti.git",
+ "https://github.com/tube42/marm.git",
  "https://github.com/tuple.git",
  "https://github.com/type_erasure.git",
  "https://github.com/type_index.git",
  "https://github.com/type_traits.git",
  "https://github.com/typeof.git",
- "https://github.com/ubawurinna/freetype-windows-binaries.git",
  "https://github.com/ublas.git",
+ "https://github.com/undbsd/rcon-dotnet.git",
  "https://github.com/units.git",
  "https://github.com/unordered.git",
+ "https://github.com/urho3d/AngelScript.git",
  "https://github.com/usineur/android-newraw.git",
  "https://github.com/utility.git",
  "https://github.com/uuid.git",
  "https://github.com/variant.git",
  "https://github.com/variant2.git",
+ "https://github.com/vegastrike/Assets-Production.git",
+ "https://github.com/viaduck/mariadbpp.git",
+ "https://github.com/vilbeyli/D3D12MemoryAllocator.git",
+ "https://github.com/vilbeyli/D3DX12.git",
+ "https://github.com/vilbeyli/VQModels.git",
+ "https://github.com/vilbeyli/VQUtils.git",
+ "https://github.com/vircadia-web.git",
  "https://github.com/vmd.git",
+ "https://github.com/vpiotr/decimal_for_cpp.git",
  "https://github.com/wave.git",
  "https://github.com/whoozle/clunk.git",
  "https://github.com/winapi.git",
  "https://github.com/wmcbrine/PDCurses.git",
+ "https://github.com/wolfpld/tracy.git",
  "https://github.com/wxWidgets/Catch.git",
  "https://github.com/wxWidgets/libexpat.git",
  "https://github.com/wxWidgets/libjpeg-turbo.git",
  "https://github.com/wxWidgets/libpng.git",
  "https://github.com/wxWidgets/libtiff.git",
  "https://github.com/wxWidgets/nanosvg.git",
+ "https://github.com/wxWidgets/pcre.git",
  "https://github.com/wxWidgets/zlib.git",
+ "https://github.com/xLAva/OculusSDK.git",
+ "https://github.com/xiph/ogg.git",
+ "https://github.com/xiph/theora.git",
  "https://github.com/xiph/vorbis.git",
  "https://github.com/xpressive.git",
  "https://github.com/yap.git",
+ "https://github.com/youkaicountry/yknyttlib.git",
+ "https://github.com/yse/easy_profiler.git",
  "https://github.com/zaps166/NFSIISE-ASM.git",
  "https://github.com/zaps166/NFSIISE-CPP.git",
  "https://github.com/zeromq/libzmq.git",
  "https://github.com/zeux/pugixml.git",
  "https://gitlab.axiodl.com/amuse.git",
- "https://gitlab.axiodl.com/hecl-gui.git",
- "https://gitlab.axiodl.com/hecl.git",
+ "https://gitlab.axiodl.com/boo.git",
  "https://gitlab.axiodl.com/jbus.git",
  "https://gitlab.axiodl.com/kabufuda.git",
+ "https://gitlab.axiodl.com/libAthena/athena.git",
+ "https://gitlab.axiodl.com/libSquish.git",
+ "https://gitlab.axiodl.com/libjpeg-turbo.git",
  "https://gitlab.axiodl.com/nod.git",
- "https://gitlab.axiodl.com/specter.git",
  "https://gitlab.axiodl.com/tinyxml2.git",
- "https://gitlab.axiodl.com/urde-translations.git",
+ "https://gitlab.axiodl.com/zeus.git",
+ "https://gitlab.com/Dringgstein/cosmos.git",
+ "https://gitlab.com/Grumbel/uitest.git",
+ "https://gitlab.com/Windstille/wstsound.git",
+ "https://gitlab.com/argparser/argparser.git",
  "https://gitlab.com/cyberegoorg/CETech-externals.git",
  "https://gitlab.com/cyberegoorg/CETech-icons-breeze.git",
+ "https://gitlab.com/grumbel/cmake-modules.git",
+ "https://gitlab.com/grumbel/geomcpp.git",
+ "https://gitlab.com/grumbel/priocpp.git",
+ "https://gitlab.com/grumbel/strutcpp.git",
+ "https://gitlab.com/grumbel/xdgcpp.git",
  "https://gitlab.com/hydren/fgeal",
  "https://gitlab.com/hydren/futil",
  "https://gitlab.com/hydren/libgeramun",
  "https://gitlab.com/lierolibre/gvl.git",
  "https://gitlab.com/lierolibre/lierolibre-data.git",
  "https://gitlab.com/lierolibre/lierolibre-w32depends.git",
- "https://gitlab.com/pingus/external/xdg.git",
- "https://invent.kde.org/kde/rcharactersheet.git",
- "https://invent.kde.org/kde/rolisteam-common.git",
- "https://invent.kde.org/kde/rolisteam-diceparser.git",
- "https://invent.kde.org/kde/rolisteam-plugins.git"
+ "https://gitlab.com/logmich/logmich.git",
+ "https://gitlab.freedesktop.org/freetype/freetype",
+ "https://gitlab.freedesktop.org/freetype/freetype.git",
+ "https://invent.kde.org/rolisteam/rcharactersheet.git",
+ "https://invent.kde.org/rolisteam/rolisteam-common.git",
+ "https://invent.kde.org/rolisteam/rolisteam-diceparser.git",
+ "https://invent.kde.org/rolisteam/rolisteam-plugins.git",
+ "https://luajit.org/git/luajit-2.0.git"
 ]

code/archives.json

@@ -616,6 +616,7 @@
   "https://github.com/clintbellanger/flare-game.git",
   "https://github.com/clintbellanger/heroine-dusk.git",
   "https://github.com/cocos2d/cocos2d-x.git",
+  "https://github.com/codenamecpp/carnage3d.git",
   "https://github.com/coelckers/Raze.git",
   "https://github.com/coelckers/gzdoom.git",
   "https://github.com/collinhover/kaiopua.git",

code/backlog.txt

@@ -37,6 +37,10 @@ https://futurepinball.com/
 https://gdevelop-app.com/
 https://github.blog/2014-01-06-github-game-off-ii-winners/
 https://github.blog/2021-05-19-ludum-dare-48-staff-picks/
+https://github.com/codenamecpp/carnage3d#other-gta-12-re-implementations
+https://github.com/Razakhel/RaZ
+https://github.com/samdauwe/BabylonCpp
+https://github.com/SasLuca/rayfork
 https://libregamewiki.org/Alexei:_Part_IX
 https://libregamewiki.org/CameliaGirls
 https://libregamewiki.org/Childish_Cannoneer
@@ -66,6 +70,7 @@ https://web.archive.org/web/20170714065728/http://users.olis.net.au/zel/
 https://web.archive.org/web/20171228172756/http://www.oletus.fi/static/whichwayisup/
 https://www.adventuregamestudio.co.uk/
 https://www.b3dgs.com/v7/page.php?lang=en&section=projects
+https://www.babylonjs.com/
 https://www.bennugd.org/
 https://www.bsframework.io/
 https://www.choiceofgames.com/make-your-own-games/choicescript-intro/

code/backlog_enhanced.txt

@@ -100,6 +100,7 @@ http://www.paulscode.com/games/
 
 github collections
 
+https://github.com/collections
 https://github.com/collections/game-engines (only OS)
 https://github.com/collections/javascript-game-engines (only OS)
 https://github.com/collections/pixel-art-tools (tools)

code/maintenance_entries.py

@@ -221,8 +221,7 @@ class EntriesMaintainer:
 
     def clean_rejected(self):
         """
-
-        :return:
+        Only sorts the rejected games list file.
         """
         # sort rejected games list file
         sort_text_file(os.path.join(c.root_path, 'code', 'rejected.txt'), 'rejected games list')

code/rejected.txt

@@ -69,7 +69,7 @@ GNOME Games (): Has been separated into individual packages. Will add them separ
 GPSFish (https://gps.tanaka.ecc.u-tokyo.ac.jp/gpsshogi/index.php?GPSFish): Too special
 Grow (https://sourceforge.net/projects/grow/): Insufficient content
 GRUB Invaders (http://www.erikyyy.de/invaders/): Requires use of the GRUB bootloader (also not really original)
-Heroes: Rebirth (
+Heroes: Rebirth (): ?
 Herzog 3D (http://herzog3d.sourceforge.net/, https://sourceforge.net/p/herzog3d/code/HEAD/tree/): Very early development
 HistoryLine (https://github.com/oliverdb/Historyline): Very early development, no license information
 Homeworld SDL (https://github.com/aheadley/homeworld): Not open source compliant license (see https://github.com/aheadley/homeworld/blob/master/README)

code/synchronize_awesome_lists.py

@@ -0,0 +1,69 @@
+"""
+Synchronizes with awesome lists from
+"""
+
+import re
+import requests
+from utils import osg, osg_rejected
+
+matcher = re.compile(r'\[(.*)?\]\((.*?)\) - (.*)')  # general structure: - [title](link) - description
+
+if __name__ == "__main__":
+    
+    # read rejected
+    rejected = osg_rejected.read_rejected_file()
+
+    # read awesome list
+    awesome_list = 'https://raw.githubusercontent.com/radek-sprta/awesome-game-remakes/master/README.md'
+    print('read {}'.format(awesome_list))
+    r = requests.get(awesome_list, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}, timeout=20,
+                     allow_redirects=True)
+    if r.status_code != requests.codes.ok:
+        raise RuntimeError('Cannot download awesome list.')
+    text = r.text
+    text = text.split('\n##')[2:]
+    entries = []
+    for items in text:
+        items = items.split('\n')
+        category = items[0].strip()
+        items = [item for item in items[1:] if item.startswith('-')]
+        for item in items:
+            matches = matcher.findall(item)[0]  # we know it will be exactly one
+            title = matches[0]
+            url = matches[1]
+            description = matches[2]
+            entries.append({'Title': title, 'URL': url, 'Description': description, 'Category': category})
+
+    # a bit of statistics about this awesome list
+    print('contains {} entries in {} categories'.format(len(entries), len(text)))
+    n = [0, 0]
+    for entry in entries:
+        if entry['URL'].startswith('https://github.com/'):
+            n[0] += 1
+        else:
+            n[1] += 1
+    print('{} links to Github, {} links not to Github'.format(*n))
+
+    # remove those that are in our rejected list
+    rejected_titles = [x['Title'] for x in rejected]
+    entries = [entry for entry in entries if entry['Title'] not in rejected_titles]
+
+    # read our database
+    our_entries = osg.read_entries()
+    print('{} entries read (osgl)'.format(len(our_entries)))
+
+    # go through this awesome list entries one by one and compare to our list
+    for entry in entries:
+        # go through our entries
+        similar_entries = []
+        for our_entry in our_entries:
+            title_equal = entry['Title'] == our_entry['Title']
+            url = entry['URL']
+            url_present = any(url in x for x in our_entry['Home']) or any(url in x for x in our_entry.get('Code repository', []))
+            if title_equal or url_present:
+                similar_entries.append(our_entry)
+        if not similar_entries:
+            print('Unknown entry "{}" {} - {} - {}'.format(entry['Title'], entry['URL'], entry['Category'], entry['Description']))
+
+
+

code/utils/osg_rejected.py

@@ -0,0 +1,44 @@
+"""
+Handling of rejected file information.
+"""
+
+import os
+import re
+from utils import constants as c, utils as u
+
+matcher = re.compile(r'(.*)? \((.*)?\): (.*)')  # general structure: name (link, link): description
+
+
+def read_rejected_file():
+    """
+
+    :return:
+    """
+    rejected_file = os.path.join(c.root_path, 'code', 'rejected.txt')
+    text = u.read_text(rejected_file)
+    rejected = []
+    for line in text.split('\n'):
+        print(line)
+        matches = matcher.findall(line)[0]  # we know there will be exactly one match on every line
+        name = matches[0].strip()
+        links = matches[1].split(',')
+        links = [link.strip() for link in links]
+        description = matches[2].strip()
+        rejected.append({'Title': name, 'URLs': links, 'Description': description})
+    return rejected
+
+
+def write_rejected_file(rejected):
+    """
+
+    :param rejected:
+    """
+    # sort by name
+    rejected.sort(key=lambda x: str.casefold(x[0]))
+    # expand single items
+    rejected = ['{} ({}): {}'.format(item[0], ', '.join(item[1]), item[2]) for item in rejected]
+    # join with newlines
+    rejected = '\n'.join(rejected)
+    # write to file
+    rejected_file = os.path.join(c.root_path, 'code', 'rejected.txt')
+    u.write_text(rejected_file, rejected)