a few more conversions

tools/archives.json

@@ -27,6 +27,8 @@
   "https://git.code.sf.net/p/epicheroes/code",
   "https://git.code.sf.net/p/erebusrpg/code",
   "https://git.code.sf.net/p/fillets/code-fillets-ng",
+  "https://git.code.sf.net/p/fillets/code-fillets_data",
+  "https://git.code.sf.net/p/fillets/code-fillets_web",
   "https://git.code.sf.net/p/flightgear/flightgear",
   "https://git.code.sf.net/p/freecol/git",
   "https://git.code.sf.net/p/freelords/git",
@@ -144,6 +146,7 @@
   "https://github.com/TVTower/TVTower.git",
   "https://github.com/TobiasBielefeld/Simple-Solitaire.git",
   "https://github.com/Trilarion/D-Fend-Reloaded.git",
+  "https://github.com/Trilarion/gltron.git",
   "https://github.com/Trilarion/mpango.git",
   "https://github.com/Trilarion/scorched3d.git",
   "https://github.com/Trilarion/spacetraderjava.git",

tools/conversions/dfend_reloaded_source_releases_to_git.py

@@ -3,13 +3,9 @@ Converts the source releases of D-Fend Reloaded to a Git.
 """
 
 import sys
-import os
-import shutil
-import zipfile
 import datetime
-import subprocess
 import re
-import time
+from utils.utils import *
 
 
 def subprocess_run(cmd):
@@ -24,32 +20,6 @@ def subprocess_run(cmd):
     else:
         print('  output: {}'.format(result.stdout.decode('ascii')))
 
-
-def unzip(zip_file, destination_directory):
-    dirs = {}
-
-    with zipfile.ZipFile(zip_file, 'r') as zip:
-        for info in zip.infolist():
-            name, date_time = info.filename, info.date_time
-            name = os.path.join(destination_directory, name)
-            zip.extract(info, destination_directory)
-
-            # still need to adjust the dt o/w item will have the current dt
-            date_time = time.mktime(info.date_time + (0, 0, -1))
-
-            if os.path.isdir(name):
-                # changes to dir dt will have no effect right now since files are
-                # being created inside of it; hold the dt and apply it later
-                dirs[name] = date_time
-            else:
-                os.utime(name, (date_time, date_time))
-
-    # done creating files, now update dir dt
-    for name in dirs:
-       date_time = dirs[name]
-       os.utime(name, (date_time, date_time))
-
-
 def single_release(zip):
     """
 
@@ -106,23 +76,6 @@ def single_release(zip):
     subprocess_run(['git', 'commit', '--message={}'.format(message), '--author={}'.format(author), '--date={}'.format(original_date)])
 
 
-def recreate_directory(path):
-    """
-
-    """
-    if os.path.isdir(path):
-        shutil.rmtree(path)
-    for attempts in range(10):
-        try:
-            os.mkdir(path)
-        except PermissionError:
-            time.sleep(0.1)
-            continue
-        else:
-            break
-    else:
-        raise RuntimeError()
-
 if __name__ == "__main__":
 
     # general properties

tools/git_statistics.py

@@ -7,3 +7,44 @@ takes all gits that we have in the list and checks the master branch out, then c
 
 uses git log --format="%an, %at, %cn, %ct" --all ti get commits, committers and times (as unix time stamp)
 """
+
+import json
+from utils.utils import *
+
+if __name__ == "__main__":
+
+    # paths
+    file_path  = os.path.realpath(os.path.dirname(__file__))
+    archives_path = os.path.join(file_path, 'git_repositories.json')
+    temp_path = os.path.join(file_path, 'temp')
+
+    # get git archives
+    text = read_text(archives_path)
+    archives = json.loads(text)
+    print('process {} git archives'.format(len(archives)))
+
+    # loop over them
+    for count, archive in enumerate(archives, 1):
+
+        # printer iteration info
+        print('{}/{} - {}'.format(count, len(archives), archive))
+
+        # recreate temp folder
+        recreate_directory(temp_path)
+        os.chdir(temp_path)
+
+        # clone git in temp folder
+        subprocess_run(["git", "clone", "--mirror", archive, temp_path])
+
+        # get commits, etc. info
+        info = subprocess_run(["git", "log", '--format="%an, %at, %cn, %ct"'])
+
+        info = info.split('\n')
+        info = info[:-1] # last line is empty
+        number_commits = len(info)
+
+        info = [x.split(', ') for x in info]
+        commiters = set([x[0] for x in info])
+
+        print(' commits: {}, commiters {}'.format(number_commits, len(commiters)))
+

tools/maintenance.py

@@ -822,8 +822,8 @@ def export_primary_code_repositories_json():
     unconsumed_entries = []
 
     # for every entry filter those that are known git repositories (add additional repositories)
+    field = 'code repository-raw'
     for info in infos:
-        field = 'code repository-raw'
         # if field 'Code repository' is available
         if field in info:
             consumed = False
@@ -878,6 +878,34 @@ def export_primary_code_repositories_json():
     write_text(json_path, text)
 
 
+def export_git_code_repositories_json():
+    """
+
+    """
+
+    urls = []
+    field = 'code repository'
+
+    # for every entry, get all git
+    for info in infos:
+        # if field 'Code repository' is available
+        if field in info:
+            repos = info[field]
+            if repos:
+                # take the first
+                repo = repos[0]
+                url = git_repo(repo)
+                if url:
+                    urls.append(url)
+
+    # sort them alphabetically (and remove duplicates)
+    urls.sort()
+
+    # write them to tools/git
+    json_path = os.path.join(root_path, 'tools', 'git_repositories.json')
+    text = json.dumps(urls, indent=1)
+    write_text(json_path, text)
+
 if __name__ == "__main__":
 
     # paths
@@ -905,5 +933,8 @@ if __name__ == "__main__":
     # collect list of primary code repositories
     export_primary_code_repositories_json()
 
+    # collect list of git code repositories (only one per project) for git_statistics script
+    # export_git_code_repositories_json()
+
     # check external links (only rarely)
     # check_validity_external_links()

tools/utils/utils.py

@@ -9,6 +9,8 @@ import tarfile
 import time
 import urllib.request
 import zipfile
+import errno
+import stat
 
 
 def read_text(file):
@@ -194,6 +196,14 @@ def download_url(url, destination):
             shutil.copyfileobj(response, f)
 
 
+def handleRemoveReadonly(func, path, exc):
+    """
+    Necessary on Windows. See https://stackoverflow.com/questions/1889597/deleting-directory-in-python
+    """
+    os.chmod(path, stat.S_IWRITE)
+    func(path)
+
+
 def git_clear_path(git_path):
     """
     Clears all in a path except the '.git' directory
@@ -204,6 +214,56 @@ def git_clear_path(git_path):
             continue
         item = os.path.join(git_path, item)
         if os.path.isdir(item):
-            shutil.rmtree(item)
+            shutil.rmtree(item, onerror=handleRemoveReadonly)
         else:
             os.remove(item)
+
+
+def recreate_directory(path):
+    """
+    Recreates a directory (deletes before if existing)
+    """
+    if os.path.isdir(path):
+        shutil.rmtree(path, onerror=handleRemoveReadonly)
+    for attempts in range(10):
+        try:
+            os.mkdir(path)
+        except PermissionError:
+            time.sleep(0.1)
+            continue
+        else:
+            break
+    else:
+        raise RuntimeError()
+
+
+def unzip(zip_file, destination_directory):
+    """
+    Unzips and keeps the original modified date.
+
+    :param zip_file:
+    :param destination_directory:
+    :return:
+    """
+    dirs = {}
+
+    with zipfile.ZipFile(zip_file, 'r') as zip:
+        for info in zip.infolist():
+            name, date_time = info.filename, info.date_time
+            name = os.path.join(destination_directory, name)
+            zip.extract(info, destination_directory)
+
+            # still need to adjust the dt o/w item will have the current dt
+            date_time = time.mktime(info.date_time + (0, 0, -1))
+
+            if os.path.isdir(name):
+                # changes to dir dt will have no effect right now since files are
+                # being created inside of it; hold the dt and apply it later
+                dirs[name] = date_time
+            else:
+                os.utime(name, (date_time, date_time))
+
+    # done creating files, now update dir dt
+    for name in dirs:
+        date_time = dirs[name]
+        os.utime(name, (date_time, date_time))