AnotherFoxGuy/opensourcegames
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

streamlined python code

Browse Source
This commit is contained in:
Trilarion 2018-06-28 14:50:35 +02:00
parent f7fd84817e
commit 20a44a6694
14 changed files with 398 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/.idea
/tools/archive/**
__pycache__

2
docs/data.json
View File

@ -84,7 +84,7 @@
[
"Alien Assault Traders (<a href=\"http://www.aatraders.com/\">home</a>, <a href=\"https://github.com/Trilarion/opensourcegames/blob/master/games/strategy/alien_assault_traders.md\">entry</a>)",
"Alien Assault Traders is an online, web-based, turn-based..",
"<a href=\"https://sourceforge.net/projects/aatrade/files/AATraders%200.4x/\">Link</a>",
"<a href=\"https://sourceforge.net/projects/aatrade/files/\">Link</a>",
"Strategy",
"beta / inactive since 2009",
"online",

2
games/rpg/linleys_dungeon_crawl.md
View File

@ -4,7 +4,7 @@ _Roguelike molded in the tradition of the early greats of the genre: Rogue, Hack
- Home: http://www.dungeoncrawl.org/
- Media: https://en.wikipedia.org/wiki/Linley%27s_Dungeon_Crawl, http://nethack.wikia.com/wiki/Linley%27s_Dungeon_Crawl
- Download: http://www.dungeoncrawl.org/?d, ftp://ftp.dungeoncrawl.org/
- Download: http://www.dungeoncrawl.org/?d (ftp://ftp.dungeoncrawl.org/)
- State: mature, inactive since 2006
- Keywords: roguelike
- Code repository: https://gitlab.com/Trilarion/dungeoncrawl.git (snapshot of source releases)

2
games/statistics.md
View File

@ -1,7 +1,7 @@
[comment]: # (autogenerated content, do not edit)
# Statistics
analyzed 403 entries on 2018-06-28 10:38:05
analyzed 403 entries on 2018-06-28 14:38:52
## State

4
games/strategy/alien_assault_traders.md
View File

@ -3,10 +3,10 @@
_Alien Assault Traders is an online, web-based, turn-based strategy space trading game forked off the source of both Black Nova Traders and Nova Game Systems software and inspired by the popular BBS game of TradeWars._
- Home: http://www.aatraders.com/, https://sourceforge.net/projects/aatrade/
- Download: https://sourceforge.net/projects/aatrade/files/AATraders%200.4x/
- Download: https://sourceforge.net/projects/aatrade/files/
- State: beta, inactive since 2009
- Keywords: online
- Code repository: https://github.com/tarnus/aatraders
- Code repository: https://github.com/tarnus/aatraders, https://gitlab.com/Trilarion/aatraders.git (+)
- Code language: PHP
- Code license: GPL-2.0

0
tools/archive/README.txt → tools/README.txt
View File

18
tools/aatraders.json Normal file
View File

@ -0,0 +1,18 @@
["https://sourceforge.net/projects/aatrade/files/AATraders%200.4x/aatrade_0.40_re-release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.4x/aatrade_0.40.0_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.3x/Alien%20Assault%20Traders%200.31.0/aatrade_0.31.0_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.3x/Alien%20Assault%20Traders%200.30.3/aatrade_0.30.3_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.3x/Alien%20Assault%20Traders%200.30.2/aatrade_0.30.2_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.3x/Alien%20Assault%20Traders%200.30.1/aatrade_0.30.1_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.3x/Alien%20Assault%20Traders%200.30/aatrade_0.30_release.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.2x%20Final/AAtraders%200.21a%20Final%20Patch/AATRade_0.21a_final.zip/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.2x%20Final/AATraders%200.21%20Release/aatrade-0.21.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.2x%20Final/AATrade%200.20%20Release/aatrade-0.20.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Release%200.14.1%20w_Profiles/aatrade-0.14.1.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Release%200.14%20w_Profiles/aatrade-0.14.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Release%20v0.13/aatrade-0.13.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Recommended%20Release%20v0.12/aatrade-0.12.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Release%20v0.11/aatrade-0.11.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/AATraders%200.1x%20Final/Initial%20Release%20v0.10/aatrade-0.10.tar.gz/download",
"https://sourceforge.net/projects/aatrade/files/OldFiles/aatrade0.21final.zip/download",
"https://sourceforge.net/projects/aatrade/files/OldFiles/aatrade0.21patched.zip/download"]

156
tools/aatraders_source_release_to_git.py Normal file
View File

@ -0,0 +1,156 @@
"""
Downloads source releases from Sourceforge and puts them into a git repository
"""
import json
import datetime
from utils.utils import *
def special_aatrade_package_extraction(source):
"""
Unpacks "aatrade_packages".
"""
files = os.listdir(source)
if any([x.startswith('aatrade_package') for x in files]):
# we got the special case
print('aatrade package extraction of {}'.format(source))
# first delete all, that do not begin with the package name
for file in files:
if not file.startswith('aatrade_package'):
os.remove(os.path.join(source, file))
# second extract all those with are left, removing them too
files = os.listdir(source)
for file in files:
try:
extract_archive(os.path.join(source, file), source, 'tar')
except:
extract_archive(os.path.join(source, file), source, 'zip')
os.remove(os.path.join(source, file))
if __name__ == '__main__':
# base path is the directory containing this file
base_path = os.path.abspath(os.path.dirname(__file__))
print('base path={}'.format(base_path))
# recreate archive path
archive_path = os.path.join(base_path, 'downloads')
if not os.path.exists(archive_path):
os.mkdir(archive_path)
# load source releases urls
with open(os.path.join(base_path, 'aatraders.json'), 'r') as f:
urls = json.load(f)
print('will process {} urls'.format(len(urls)))
if len(urls) != len(set(urls)):
raise RuntimeError("urls list contains duplicates")
# determine file archives from urls
archives = [x.split('/')[-2] for x in urls]
if len(archives) != len(set(archives)):
raise RuntimeError("files with duplicate archives, cannot deal with that")
# determine version from file name
versions = [determine_archive_version_generic(x, leading_terms=['aatrade_', 'aatrade-', 'aatrade'], trailing_terms=['.zip', '.tar.gz', '_release']) for x in archives]
for version in versions:
print(version)
# extend archives to full paths
archives = [os.path.join(archive_path, x) for x in archives]
# download them
print('download source releases')
for url, destination in zip(urls, archives):
# only if not yet existing
if os.path.exists(destination):
continue
# download
print(' download {}'.format(os.path.basename(destination)))
download_url(url, destination)
# extract them
print('extract downloaded archives')
extracted_archives = [x + '-extracted' for x in archives]
for archive, extracted_archive in zip(archives, extracted_archives):
print(' extract {}'.format(os.path.basename(archive)))
# only if not yet existing
if os.path.exists(extracted_archive):
continue
os.mkdir(extracted_archive)
# extract
extract_archive(archive, extracted_archive, detect_archive_type(archive))
# go up in unzipped archives until the very first non-empty folder
extracted_archives = [strip_wrapped_folders(x) for x in extracted_archives]
# special 'aatrade_packageX' treatment
for extracted_archive in extracted_archives:
special_aatrade_package_extraction(extracted_archive)
# calculate size of folder
sizes = [folder_size(x) for x in extracted_archives]
# determine date
dates = [determine_latest_last_modified_date(x) for x in extracted_archives]
dates_strings = [datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d') for x in dates]
# if len(dates_strings) != len(set(dates_strings)):
# raise RuntimeError("Some on the same day, cannot cope with that")
# gather all important stuff in one list and sort by dates and throw those out where size is not in range
db = list(zip(urls, extracted_archives, versions, dates, dates_strings, sizes))
db.sort(key=lambda x:x[3])
size_range = [5e6, float("inf")] # set to None if not desired
if size_range:
db = [x for x in db if size_range[0] <= x[5] <= size_range[1]]
print('proposed order')
for url, _, version, _, date, size in db:
print(' date={} version={} size={}'.format(date, version, size))
# git init
git_path = os.path.join(base_path, 'aatrade')
if os.path.exists(git_path):
shutil.rmtree(git_path)
os.mkdir(git_path)
os.chdir(git_path)
subprocess_run(['git', 'init'])
subprocess_run(['git', 'config', 'user.name', 'Trilarion'])
subprocess_run(['git', 'config', 'user.email', 'Trilarion@users.noreply.gitlab.com'])
# now process revision by revision
print('process revisions')
git_author = 'akapanamajack, tarnus <akapanamajack_tarnus@user.sourceforge.net>'
for url, archive_path, version, _, date, _ in db:
print(' process version={}'.format(version))
# clear git path without deleting .git
print(' clear git')
for item in os.listdir(git_path):
# ignore '.git
if item == '.git':
continue
item = os.path.join(git_path, item)
if os.path.isdir(item):
shutil.rmtree(item)
else:
os.remove(item)
# copy unpacked source files to git path
print('copy to git')
copy_tree(archive_path, git_path)
# update the git index (add unstaged, remove deleted, ...)
print('git add')
os.chdir(git_path)
subprocess_run(['git', 'add', '--all'])
# perform the commit
print('git commit')
os.chdir(git_path)
message = 'version {} ({}) on {}'.format(version, url, date)
print(' message "{}"'.format(message))
subprocess_run(['git', 'commit', '--message={}'.format(message), '--author={}'.format(git_author), '--date={}'.format(date)])

1
tools/archive/archives.json → tools/archives.json
View File

@ -298,6 +298,7 @@
"https://github.com/xesf/twin-e.git",
"https://github.com/xoreos/xoreos.git",
"https://github.com/zaki/irrlicht.git",
"https://gitlab.com/Trilarion/aatraders.git",
"https://gitlab.com/Trilarion/antichess.git",
"https://gitlab.com/Trilarion/aplanetsrevenge.git",
"https://gitlab.com/Trilarion/attal.git",

27
tools/maintenance.py
View File

@ -15,36 +15,11 @@ import http.client
import datetime
import json
import textwrap
from utils.utils import *
TOC = '_toc.md'
def read_text(file):
"""
Reads a whole text file (UTF-8 encoded).
"""
with open(file, mode='r', encoding='utf-8') as f:
text = f.read()
return text
def read_first_line(file):
"""
Convenience function because we only need the first line of a category overview really.
"""
with open(file, mode='r', encoding='utf-8') as f:
line = f.readline()
return line
def write_text(file, text):
"""
Writes a whole text file (UTF-8 encoded).
"""
with open(file, mode='w', encoding='utf-8') as f:
f.write(text)
def get_category_paths():
"""
Returns all sub folders of the games path.

90
tools/phaos_source_release_to_git.py
View File

@ -2,89 +2,9 @@
Downloads source releases from Sourceforge and puts them into a git repository
"""
import os
import shutil
import urllib.request
import json
import time
import zipfile
import subprocess
import datetime
import distutils.dir_util
import sys
def determine_version(name):
# to lower case
name = name.lower()
# cut leading terms
terms = ['phaos-', 'phaos', 'pv']
for t in terms:
if name.startswith(t):
name = name[len(t):]
# cut trailing '.zip'
t = '.zip'
if name.endswith(t):
name = name[:-len(t)]
return name
def determine_last_modified_date(folder):
latest_last_modified = 0
for dirpath, dirnames, filenames in os.walk(folder):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
lastmodified = os.path.getmtime(filepath)
if lastmodified > latest_last_modified:
latest_last_modified = lastmodified
return latest_last_modified
def unzip_keep_last_modified(archive, destination):
"""
Assuming that destination is a directory and already existing.
"""
with zipfile.ZipFile(archive, 'r') as zip:
# zip.extractall(destination)
for zip_entry in zip.infolist():
name, date_time = zip_entry.filename, zip_entry.date_time
date_time = time.mktime(date_time + (0, 0, -1))
zip.extract(zip_entry, destination)
os.utime(os.path.join(destination, name), (date_time, date_time))
def strip_wrapping(folder):
names = os.listdir(folder)
while len(names) == 1:
folder = os.path.join(folder, names[0])
names = os.listdir(folder)
return folder
def copy_tree(source, destination):
# this gave an FileNotFoundError: [Errno 2] No such file or directory: '' on Windows
# distutils.dir_util.copy_tree(archive_path, git_path)
for dirpath, dirnames, filenames in os.walk(source):
# first create all the directory on destination
directories_to_be_created = [os.path.join(destination, os.path.relpath(os.path.join(dirpath, x), source)) for x in dirnames]
for directory in directories_to_be_created:
os.makedirs(directory, exist_ok=True)
# second copy all the files
filepaths_source = [os.path.join(dirpath, x) for x in filenames]
filepaths_destination = [os.path.join(destination, os.path.relpath(x, source)) for x in filepaths_source]
for src, dst in zip(filepaths_source, filepaths_destination):
shutil.copyfile(src, dst)
def subprocess_run(cmd):
"""
"""
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode:
print("error {} in call {}".format(result.returncode, cmd))
print(result.stderr.decode('ascii'))
sys.exit(-1)
else:
print(' output: {}'.format(result.stdout.decode('ascii')))
from utils.utils import *
if __name__ == '__main__':
@ -113,7 +33,7 @@ if __name__ == '__main__':
raise RuntimeError("files with duplicate archives, cannot deal with that")
# determine version from file name
versions = [determine_version(x) for x in archives]
versions = [determine_archive_version_generic(x, leading_terms=['phaos-', 'phaos', 'pv'], trailing_terms=['zip']) for x in archives]
# for version in versions:
# print(version)
@ -146,10 +66,10 @@ if __name__ == '__main__':
unzip_keep_last_modified(archive, unzipped_archive)
# go up in unzipped archives until the very first non-empty folder
unzipped_archives = [strip_wrapping(x) for x in unzipped_archives]
unzipped_archives = [strip_wrapped_folders(x) for x in unzipped_archives]
# determine date
dates = [determine_last_modified_date(x) for x in unzipped_archives]
dates = [determine_latest_last_modified_date(x) for x in unzipped_archives]
dates_strings = [datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d') for x in dates]
# if len(dates_strings) != len(set(dates_strings)):
# raise RuntimeError("Some on the same day, cannot cope with that")
@ -168,6 +88,8 @@ if __name__ == '__main__':
os.mkdir(git_path)
os.chdir(git_path)
subprocess_run(['git', 'init'])
subprocess_run(['git', 'config', 'user.name', 'Trilarion'])
subprocess_run(['git', 'config', 'user.email', 'Trilarion@users.noreply.gitlab.com'])
# now process revision by revision
print('process revisions')

48
tools/archive/update.py → tools/update.py
View File

@ -11,20 +11,10 @@
Note: May need to set http.postBuffer (https://stackoverflow.com/questions/17683295/git-bash-error-rpc-failed-result-18-htp-code-200b-1kib-s)
"""
import os
import json
import subprocess
import time
from utils.utils import *
def read_text(file):
"""
Reads a whole text file (UTF-8 encoded).
"""
with open(file, mode='r', encoding='utf-8') as f:
text = f.read()
return text
def derive_folder_name(url, replaces):
sanitize = lambda x: x.replace('/', '.')
for service in replaces:
@ -52,16 +42,12 @@ def git_folder_name(url):
def git_clone(url, folder):
result = subprocess.run(["git", "clone", "--mirror", url, folder])
if result.returncode:
print(result)
subprocess_run(["git", "clone", "--mirror", url, folder])
def git_update(folder):
os.chdir(folder)
result = subprocess.run(["git", "fetch", "--all"])
if result.returncode:
print(result)
subprocess_run(["git", "fetch", "--all"])
def svn_folder_name(url):
@ -72,15 +58,12 @@ def svn_folder_name(url):
def svn_clone(url, folder):
result = subprocess.run(["svn", "checkout", url, folder])
if result.returncode:
print(result)
subprocess_run(["svn", "checkout", url, folder])
def svn_update(folder):
os.chdir(folder)
result = subprocess.run(["svn", "update"])
if result.returncode:
print(result)
subprocess_run(["svn", "update"])
def hg_folder_name(url):
@ -93,16 +76,12 @@ def hg_folder_name(url):
def hg_clone(url, folder):
result = subprocess.run(["hg", "clone", url, folder])
if result.returncode:
print(result)
subprocess_run(["hg", "clone", url, folder])
def hg_update(folder):
os.chdir(folder)
result = subprocess.run(['hg', 'pull', '-u'])
if result.returncode:
print(result)
subprocess_run(['hg', 'pull', '-u'])
def bzr_folder_name(url):
@ -113,21 +92,17 @@ def bzr_folder_name(url):
def bzr_clone(url, folder):
result = subprocess.run(['bzr', 'branch', url, folder])
if result.returncode:
print(result)
subprocess_run(['bzr', 'branch', url, folder])
def bzr_update(folder):
os.chdir(folder)
result = subprocess.run(['bzr', 'pull'])
if result.returncode:
print(result)
subprocess_run(['bzr', 'pull'])
def run(type, urls):
print('update {} {} archives'.format(len(urls), type))
base_folder = os.path.join(root_folder, type)
base_folder = os.path.join(archive_folder, type)
if not os.path.exists(base_folder):
os.mkdir(base_folder)
@ -192,6 +167,7 @@ if __name__ == '__main__':
# get this folder
root_folder = os.path.realpath(os.path.dirname(__file__))
archive_folder = os.path.join(root_folder, 'archive')
# read archives.json
text = read_text(os.path.join(root_folder, 'archives.json'))

0
tools/utils/__init__.py Normal file
View File

191
tools/utils/utils.py Normal file
View File

@ -0,0 +1,191 @@
"""
Utilities for the tools.
"""
import sys
import os
import time
import zipfile
import tarfile
import subprocess
import shutil
import urllib.request
def read_text(file):
"""
Reads a whole text file (UTF-8 encoded).
"""
with open(file, mode='r', encoding='utf-8') as f:
text = f.read()
return text
def read_first_line(file):
"""
Convenience function because we only need the first line of a category overview really.
"""
with open(file, mode='r', encoding='utf-8') as f:
line = f.readline()
return line
def write_text(file, text):
"""
Writes a whole text file (UTF-8 encoded).
"""
with open(file, mode='w', encoding='utf-8') as f:
f.write(text)
def determine_archive_version_generic(name, leading_terms, trailing_terms):
"""
Given an archive file name, tries to get version information. Generic version that can cut off leading and trailing
terms and converts to lower case. Give the most special terms first in the list. As many cut offs as possible are
performed.
"""
# to lower case
name = name.lower()
# cut leading terms
for t in leading_terms:
if name.startswith(t):
name = name[len(t):]
# cut trailing terms
for t in trailing_terms:
if name.endswith(t):
name = name[:-len(t)]
return name
def unzip_keep_last_modified(archive, destination):
"""
Unzips content of a zip file archive into the destination directory keeping the last modified file property as
it was in the zip archive.
Assumes that destination is an existing directory path.
"""
with zipfile.ZipFile(archive, 'r') as zip:
# zip.extractall(destination) # does not keep the last modified property
for zip_entry in zip.infolist():
name, date_time = zip_entry.filename, zip_entry.date_time
date_time = time.mktime(date_time + (0, 0, -1))
zip.extract(zip_entry, destination)
os.utime(os.path.join(destination, name), (date_time, date_time))
def detect_archive_type(name):
"""
Tries to guess which type an archive is.
"""
# test for tar
tar_endings = ['.tbz2', '.tar.gz']
for ending in tar_endings:
if name.endswith(ending):
return 'tar'
# test for zip
zip_endings = ['.zip', '.jar']
for ending in zip_endings:
if name.endswith(ending):
return 'zip'
# unknown
return None
def folder_size(path):
size = 0
for dirpath, dirnames, filenames in os.walk(path):
for file in filenames:
size += os.path.getsize(os.path.join(dirpath, file))
return size
def extract_archive(source, destination, type):
"""
Extracts a zip, tar, ... to a destination path.
Type may result from detect_archive_type().
"""
if type == 'tar':
tar = tarfile.open(source, 'r')
tar.extractall(destination)
elif type == 'zip':
unzip_keep_last_modified(source, destination)
def strip_wrapped_folders(folder):
"""
If a folder only contains a single sub-folder and nothing else, descends this way as much as possible.
Assumes folder is a directory.
"""
while True:
entries = list(os.scandir(folder))
if len(entries) == 1 and entries[0].is_dir():
folder = entries[0].path
else:
break
return folder
def determine_latest_last_modified_date(folder):
"""
Given a folder, recursively searches all files in this folder and all sub-folders and memorizes the latest
"last modified" date of all these files.
"""
latest_last_modified = 0
for dirpath, dirnames, filenames in os.walk(folder):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
lastmodified = os.path.getmtime(filepath)
if lastmodified > latest_last_modified:
latest_last_modified = lastmodified
return latest_last_modified
def subprocess_run(cmd):
"""
Runs a cmd via subprocess and displays the std output in case of success or the std error output in case of failure
where it also stops execution.
"""
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode:
print("error {} in call {}".format(result.returncode, cmd))
print(result.stderr.decode('ascii'))
sys.exit(-1)
else:
print(' output: {}'.format(result.stdout.decode('ascii')))
def copy_tree(source, destination):
"""
Copies the full content of one directory into another avoiding the use of distutils.di_util.copy_tree because that
can give unwanted errors on Windows (probably related to symlinks).
"""
# this gave an FileNotFoundError: [Errno 2] No such file or directory: '' on Windows
# distutils.dir_util.copy_tree(archive_path, git_path)
for dirpath, dirnames, filenames in os.walk(source):
# first create all the directory on destination
directories_to_be_created = [os.path.join(destination, os.path.relpath(os.path.join(dirpath, x), source)) for x in dirnames]
for directory in directories_to_be_created:
os.makedirs(directory, exist_ok=True)
# second copy all the files
filepaths_source = [os.path.join(dirpath, x) for x in filenames]
filepaths_destination = [os.path.join(destination, os.path.relpath(x, source)) for x in filepaths_source]
for src, dst in zip(filepaths_source, filepaths_destination):
shutil.copyfile(src, dst)
def download_url(url, destination):
"""
Using urllib.request downloads from an url to a destination. Destination will be a file.
Waits one second before, trying to be nice.
"""
time.sleep(1) # we are nice
with urllib.request.urlopen(url) as response:
with open(destination, 'wb') as f:
shutil.copyfileobj(response, f)