archive update (with location specified in local-config.ini) and small updates of links

code/archive_update.py

@@ -11,21 +11,28 @@ TODO are really all existing branches cloned and pulled? (see https://stackoverf
 TODO Sourceforge git clone may not work all the time (restarting the script sometimes helps..)
 
 Note: May need to set http.postBuffer (https://stackoverflow.com/questions/17683295/git-bash-error-rpc-failed-result-18-htp-code-200b-1kib-s)
+
+
+For repositories
+see https://serverfault.com/questions/544156/git-clone-fail-instead-of-prompting-for-credentials
 """
 
 import json
 
 from utils.utils import *
 from utils.archive import *
+import utils.constants as c
 
 
 def git_clone(url, folder):
+    # subprocess_run(["git", "clone", "--mirror", url, folder], shell=True, env={'GIT_TERMINAL_PROMPT': '0'})
     subprocess_run(["git", "clone", "--mirror", url, folder])
 
 
 def git_update(folder):
     os.chdir(folder)
-    subprocess_run(["git", "fetch", "--all"])
+    # subprocess_run(["git", "fetch", "--all"],  shell=True, env={'GIT_TERMINAL_PROMPT': '0'})
+    subprocess_run(["git", "fetch", "--all"], display=False)
 
 
 def svn_folder_name(url):
@@ -69,6 +76,9 @@ def run_update(type, urls, type_folder=None):
     base_folder = os.path.join(archive_folder, type_folder)
     if not os.path.exists(base_folder):
         os.mkdir(base_folder)
+    unused_base_folder = os.path.join(archive_folder, type_folder + '.unused')
+    if not os.path.exists(unused_base_folder):
+        os.mkdir(unused_base_folder)
 
     # get derived folder names
     folders = [folder_name[type](url) for url in urls]
@@ -76,9 +86,12 @@ def run_update(type, urls, type_folder=None):
     # find those folders not used anymore
     existing_folders = [x for x in os.listdir(base_folder) if os.path.isdir(os.path.join(base_folder, x))]
     unused_folders = [x for x in existing_folders if x not in folders]
-    print('{} unused archives'.format(len(unused_folders)))
-    if unused_folders:
-        print(unused_folders)
+    print('{} unused archives, move to unused folder'.format(len(unused_folders)))
+    for folder in unused_folders:
+        origin = os.path.join(base_folder, folder)
+        destination = os.path.join(unused_base_folder, folder)
+        if not os.path.exists(destination):
+            shutil.move(origin, destination)
 
     # new folder, need to clone
     new_folders = [x for x in folders if x not in existing_folders]
@@ -101,7 +114,7 @@ def run_update(type, urls, type_folder=None):
         if not os.path.isdir(folder):
             print('folder not existing, wanted to update, will skip')
             continue
-        print('update {}'.format(folder[len(base_folder):]))
+        # print('update {}'.format(folder[len(base_folder):]))
         try:
             update[type](folder)
         except RuntimeError as e:
@@ -148,7 +161,9 @@ if __name__ == '__main__':
 
     # get this folder
     root_folder = os.path.realpath(os.path.dirname(__file__))
-    archive_folder = os.path.join(root_folder, 'archive')
+    archive_folder = c.get_config('archive-folder')
+    if not archive_folder:
+        raise RuntimeError('No archive folder specified.')
 
     # read archives.json
     text = read_text(os.path.join(root_folder, 'archives.json'))

code/archives.json

@@ -295,6 +295,7 @@
   "https://github.com/ObKo/OpenSR.git",
   "https://github.com/OneSleepyDev/boswars_osd.git",
   "https://github.com/OneSleepyDev/boswars_osd_archive.git",
+  "https://github.com/OnlineCop/kqlives.git",
   "https://github.com/OoliteProject/oolite.git",
   "https://github.com/OpMonTeam/OpMon-Data.git",
   "https://github.com/OpMonTeam/OpMon.git",
@@ -641,7 +642,6 @@
   "https://github.com/gregoryfenton/other-life.git",
   "https://github.com/griefly/griefly.git",
   "https://github.com/grit-engine/grit-engine.git",
-  "https://github.com/grrk-bzzt/kqlives.git",
   "https://github.com/gtheilman/RxWars.git",
   "https://github.com/guillaume-gouchon/dungeonhero.git",
   "https://github.com/guillaume-gouchon/dungeonquest.git",
@@ -792,6 +792,7 @@
   "https://github.com/nenadalm/Train.git",
   "https://github.com/nevat/abbayedesmorts-gpl.git",
   "https://github.com/nhydock/UlDunAd.git",
+  "https://github.com/nicholas-ochoa/OpenSC2K.git",
   "https://github.com/nicupavel/openpanzer.git",
   "https://github.com/nigels-com/glew.git",
   "https://github.com/nikki-and-the-robots/nikki.git",
@@ -880,7 +881,6 @@
   "https://github.com/radu124/splexhd.git",
   "https://github.com/raduprv/Eternal-Lands.git",
   "https://github.com/rafaelcastrocouto/enduro.git",
-  "https://github.com/rage8885/OpenSC2K.git",
   "https://github.com/rayjohannessen/songofalbion.git",
   "https://github.com/raysan5/raylib.git",
   "https://github.com/raysan5/rfxgen.git",
@@ -1385,7 +1385,6 @@
   "svn://svn.zoy.org/abuse/abuse/trunk"
  ],
  "hg": [
-  "http://hg.assembla.com/parpg-core",
   "http://hg.code.sf.net/p/blood/code",
   "http://hg.code.sf.net/p/directpython11/code",
   "http://hg.code.sf.net/p/fltrator/code",

code/utils/constants.py

@@ -3,6 +3,7 @@ Paths, properties.
 """
 
 import os
+import configparser
 
 # paths
 root_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
@@ -10,4 +11,16 @@ entries_path = os.path.join(root_path, 'entries')
 tocs_path = os.path.join(entries_path, 'tocs')
 code_path = os.path.join(root_path, 'code')
 
-local_properties_file = os.path.join(root_path, 'local.properties')
+local_config_file = os.path.join(root_path, 'local-config.ini')
+
+config = configparser.ConfigParser()
+config.read(local_config_file)
+
+
+def get_config(key):
+    """
+
+    :param key:
+    :return:
+    """
+    return config['general'][key]

code/utils/utils.py

@@ -146,12 +146,12 @@ def determine_latest_last_modified_date(folder):
     return latest_last_modified
 
 
-def subprocess_run(cmd, display=True):
+def subprocess_run(cmd, display=True, shell=False, env={}):
     """
     Runs a cmd via subprocess and displays the std output in case of success or the std error output in case of failure
     where it also stops execution.
     """
-    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=dict(os.environ, **env))
     if result.returncode:
         if display:
             print("error {} in call {}".format(result.returncode, cmd))