maintenance (replaced dead links, added Git repositories, code dependencies, ...)

tools/archives.json

@@ -354,7 +354,6 @@
   "https://github.com/xoreos/xoreos.git",
   "https://github.com/zaki/irrlicht.git",
   "https://github.com/zenorogue/hyperrogue.git",
-  "https://gitlab.com/Trilarion/worldbuilder.git",
   "https://gitlab.com/evol/evol-all.git",
   "https://gitlab.com/freedroid/freedroid-src.git",
   "https://gitlab.com/manaplus/manaplus.git",
@@ -362,12 +361,15 @@
   "https://gitlab.com/osgames/antichess.git",
   "https://gitlab.com/osgames/aplanetsrevenge.git",
   "https://gitlab.com/osgames/armies.git",
+  "https://gitlab.com/osgames/atanks.git",
   "https://gitlab.com/osgames/attal.git",
   "https://gitlab.com/osgames/avanor.git",
   "https://gitlab.com/osgames/blacknova.git",
   "https://gitlab.com/osgames/blitzkrieg.git",
+  "https://gitlab.com/osgames/blobwars.git",
   "https://gitlab.com/osgames/bmtactics.git",
   "https://gitlab.com/osgames/boson.git",
+  "https://gitlab.com/osgames/celestron.git",
   "https://gitlab.com/osgames/civil.git",
   "https://gitlab.com/osgames/civilwar.git",
   "https://gitlab.com/osgames/coltoo.git",
@@ -379,13 +381,16 @@
   "https://gitlab.com/osgames/deity.git",
   "https://gitlab.com/osgames/devana.git",
   "https://gitlab.com/osgames/digitalalovestory.git",
+  "https://gitlab.com/osgames/dnt.git",
   "https://gitlab.com/osgames/dragon-history.git",
   "https://gitlab.com/osgames/dragon-hunt.git",
   "https://gitlab.com/osgames/duelcommander.git",
+  "https://gitlab.com/osgames/dunedynasty.git",
   "https://gitlab.com/osgames/dungeoncrawl.git",
   "https://gitlab.com/osgames/dungeonmap.git",
   "https://gitlab.com/osgames/endlessdungeons.git",
   "https://gitlab.com/osgames/eos-game.git",
+  "https://gitlab.com/osgames/epicheroes.git",
   "https://gitlab.com/osgames/firststrike.git",
   "https://gitlab.com/osgames/freetrain.git",
   "https://gitlab.com/osgames/fujo.git",
@@ -402,6 +407,8 @@
   "https://gitlab.com/osgames/kursk.git",
   "https://gitlab.com/osgames/lechemindeladam.git",
   "https://gitlab.com/osgames/lincity.git",
+  "https://gitlab.com/osgames/lipsofsuna.git",
+  "https://gitlab.com/osgames/liquidwar6.git",
   "https://gitlab.com/osgames/lost-sky.git",
   "https://gitlab.com/osgames/low.git",
   "https://gitlab.com/osgames/machinations.git",
@@ -427,13 +434,16 @@
   "https://gitlab.com/osgames/spaceopera.git",
   "https://gitlab.com/osgames/spicetrade.git",
   "https://gitlab.com/osgames/tbots.git",
+  "https://gitlab.com/osgames/the-bubs-brothers.git",
   "https://gitlab.com/osgames/theclans.git",
   "https://gitlab.com/osgames/tuxracer.git",
   "https://gitlab.com/osgames/uaf.git",
+  "https://gitlab.com/osgames/umbra.git",
   "https://gitlab.com/osgames/ura-game.git",
   "https://gitlab.com/osgames/uwadv.git",
   "https://gitlab.com/osgames/wargamer.git",
   "https://gitlab.com/osgames/watomic.git",
+  "https://gitlab.com/osgames/worldbuilder.git",
   "https://gitlab.com/osgames/worldofheroes.git",
   "https://gitlab.com/osgames/xarchon.git",
   "https://gitlab.com/osgames/xconq.git",

tools/list_python_external_imports.py

@@ -19,9 +19,9 @@ def local_module(module_base, file_path, module):
 
 if __name__ == "__main__":
 
-    system_libraries = {'__builtin__', '.', '..', '*', 'array', 'os', 'copy', 'codecs', 'collections', 'cPickle', 'datetime', 'decimal', 'email',
-        'io', 'math', 'md5', 'operator', 'random', 're', 'sha', 'shutil', 'smtplib', 'socket', 'string', 'struct', 'subprocess',
-        'sys', 'thread', 'threading', 'time', 'traceback', 'types', 'urllib', 'urllib2', 'yaml', 'yaml3', 'zlib'}
+    system_libraries = {'__builtin__', '.', '..', '*', 'argparse', 'array', 'os', 'copy', 'codecs', 'collections', 'ctypes', 'pickle', 'cPickle', 'datetime', 'decimal', 'email', 'functools',
+        'io', 'itertools', 'json', 'httplib', 'glob', 'math', 'cmath', 'heapq', 'md5', 'operator', 'random', 're', 'sha', 'shutil', 'smtplib', 'socket', 'string', 'struct', 'subprocess',
+        'sys', 'thread', 'threading', 'time', 'traceback', 'types', 'urllib', 'urllib2', 'urlparse', 'unittest', 'yaml', 'yaml3', 'zlib', 'zipfile', '__future__'}
     regex_import = re.compile(r"^\s*import (.*)", re.MULTILINE)
     regex_from = re.compile(r"^\s*from (.*) import (.*)", re.MULTILINE)
     regex_comment = re.compile(r"(#.*)$", re.MULTILINE)
@@ -32,17 +32,23 @@ if __name__ == "__main__":
     module_base = r''
 
     # get all *.py files below the root_folder
-    files = []
+    python_files = []
+    setup_files = []
     for dirpath, dirnames, filenames in os.walk(root_folder):
-        filenames = [x for x in filenames if x.endswith('.py') or x.endswith('.pyw')]
+        for file in ('setup.py', 'requirements.txt'):
+            if file in filenames:
+                setup_files.append(os.path.join(dirpath, file))
+        filenames = [x for x in filenames if x.endswith('.py') or x.endswith('.pyw') or x.endswith('.cry')]
         if filenames:
             filenames = [os.path.join(dirpath, x) for x in filenames]
-            files.extend(filenames)
-    print('found {} files'.format(len(files)))
+            python_files.extend(filenames)
+    print('found {} Python files'.format(len(python_files)))
+    if setup_files:
+        print('found setup files: {}'.format(', '.join(setup_files)))
 
     # iterate over all these files
     imports = []
-    for file in files:
+    for file in python_files:
 
         # get file path
         file_path = os.path.split(file)[0]

tools/maintenance.py

@@ -149,8 +149,8 @@ def check_validity_external_links():
                 # for each possible clause
                 for url in match:
 
-                    # if there was something
-                    if url:
+                    # if there was something (and not a sourceforge git url)
+                    if url and not url.startswith('https://git.code.sf.net/p/'):
                         try:
                             # without a special header, frequent 403 responses occur
                             req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'})

tools/utils/utils.py

@@ -15,7 +15,7 @@ def read_text(file):
     """
     Reads a whole text file (UTF-8 encoded).
     """
-    with open(file, mode='r', encoding='utf-8') as f:
+    with open(file, mode='r', encoding='utf-8', errors='ignore') as f:
         text = f.read()
     return text