fixed entries
This commit is contained in:
Trilarion
@ -1,5 +1,5 @@
|
||||
Clones and/or pulls many git repositories from the open source games entries, so that one has an archive of them.
|
||||
|
||||
Currently requires at least 55 GB space!
|
||||
Currently requires at least 60 GB space!
|
||||
|
||||
Run update.py to update the archive. Git URLs are stored in archives.json.
|
||||
@ -18,6 +18,7 @@
|
||||
"https://git.code.sf.net/p/dnt/code",
|
||||
"https://git.code.sf.net/p/dropshock/code",
|
||||
"https://git.code.sf.net/p/dunedynasty/dunedynasty",
|
||||
"https://git.code.sf.net/p/dunelegacy/code",
|
||||
"https://git.code.sf.net/p/epicheroes/code",
|
||||
"https://git.code.sf.net/p/erebusrpg/code",
|
||||
"https://git.code.sf.net/p/freecol/git",
|
||||
@ -48,6 +49,7 @@
|
||||
"https://github.com/Andrettin/Wyrmgus.git",
|
||||
"https://github.com/Argentum-Online/Argentum-Online.git",
|
||||
"https://github.com/Atlantis-PBEM/Atlantis.git",
|
||||
"https://github.com/Attnam/ivan.git",
|
||||
"https://github.com/BackupTheBerlios/netpanzer-svn.git",
|
||||
"https://github.com/CamHenlin/Roguish.git",
|
||||
"https://github.com/Circular-Studios/Dash.git",
|
||||
@ -161,6 +163,7 @@
|
||||
"https://github.com/enigma-dev/enigma-dev.git",
|
||||
"https://github.com/exult/exult.git",
|
||||
"https://github.com/fabianschuiki/OpenSkyscraper.git",
|
||||
"https://github.com/farmboy0/slashem.git",
|
||||
"https://github.com/fastrgv/AdaVenture.git",
|
||||
"https://github.com/freeciv/freeciv-web.git",
|
||||
"https://github.com/freeciv/freeciv.git",
|
||||
@ -278,7 +281,48 @@
|
||||
"https://gitlab.gnome.org/GNOME/libxml2.git",
|
||||
"https://gitorious.org/turious/turious.git"
|
||||
],
|
||||
"svn": [],
|
||||
"hg": [],
|
||||
"bzr": []
|
||||
"svn": [
|
||||
"https://svn.code.sf.net/p/ageofmagic/code/",
|
||||
"https://svn.code.sf.net/p/atomiks/code/",
|
||||
"https://svn.code.sf.net/p/begin2/code/",
|
||||
"https://svn.code.sf.net/p/catmother/code/",
|
||||
"https://svn.code.sf.net/p/crossfire/code/",
|
||||
"https://svn.code.sf.net/p/daimonin/code/",
|
||||
"https://svn.code.sf.net/p/darkcity/code/",
|
||||
"https://svn.code.sf.net/p/darkdestiny/code/",
|
||||
"https://svn.code.sf.net/p/firststrikegame/code/",
|
||||
"https://svn.code.sf.net/p/freesynd/code/",
|
||||
"https://svn.code.sf.net/p/fsc/code/",
|
||||
"https://svn.code.sf.net/p/kralovstvi/code/",
|
||||
"https://svn.code.sf.net/p/lechemindeladam/code/",
|
||||
"https://svn.code.sf.net/p/low/code/",
|
||||
"https://svn.code.sf.net/p/opencity/code/",
|
||||
"https://svn.code.sf.net/p/opengeneral/code/",
|
||||
"https://svn.code.sf.net/p/openpopulous/code/",
|
||||
"https://svn.code.sf.net/p/opensnc/code/",
|
||||
"https://svn.code.sf.net/p/pio/code/",
|
||||
"https://svn.code.sf.net/p/planeshift/code/",
|
||||
"https://svn.code.sf.net/p/solaristhegame/code/",
|
||||
"https://svn.code.sf.net/p/space-faring/code/",
|
||||
"https://svn.code.sf.net/p/spacetraderjava/code/",
|
||||
"https://svn.code.sf.net/p/speed-dreams/code/",
|
||||
"https://svn.code.sf.net/p/xu4/code/"
|
||||
],
|
||||
"hg": [
|
||||
"http://hg.asc-hq.org/hg/asc",
|
||||
"http://hg.assembla.com/parpg-core",
|
||||
"http://hg.code.sf.net/p/phantasy/code",
|
||||
"http://hg.libsdl.org/SDL",
|
||||
"https://bitbucket.org/Darthman/quad/src",
|
||||
"https://bitbucket.org/allacrost/allacrost/src",
|
||||
"https://bitbucket.org/featheredmelody/lost-sky-project-public/src",
|
||||
"https://bitbucket.org/genericcontainer/goblin-camp/src",
|
||||
"https://bitbucket.org/giszmo/glob2",
|
||||
"https://bitbucket.org/rude/love",
|
||||
"https://bitbucket.org/sumwars/sumwars-code/src"
|
||||
],
|
||||
"bzr": [
|
||||
"https://code.launchpad.net/heroesofwesnoth",
|
||||
"https://code.launchpad.net/openhomm"
|
||||
]
|
||||
}
|
||||
@ -7,6 +7,8 @@
|
||||
|
||||
TODO are really all existing branches cloned and pulled? (see https://stackoverflow.com/questions/67699/how-to-clone-all-remote-branches-in-git)
|
||||
TODO Sourceforge git clone may not work all the time (restart the script helps..)
|
||||
|
||||
Note: May need to set http.postBuffer (https://stackoverflow.com/questions/17683295/git-bash-error-rpc-failed-result-18-htp-code-200b-1kib-s)
|
||||
"""
|
||||
|
||||
import os
|
||||
@ -23,17 +25,7 @@ def read_text(file):
|
||||
text = f.read()
|
||||
return text
|
||||
|
||||
|
||||
def derive_folder_name(url):
|
||||
replaces = {
|
||||
'https://github.com': 'github',
|
||||
'https://git.code.sf.net/p': 'sourceforge',
|
||||
'https://git.tuxfamily.org': 'tuxfamily',
|
||||
'https://git.savannah.gnu.org/git': 'savannah.gnu',
|
||||
'https://gitlab.com': 'gitlab',
|
||||
'https://gitorious.org': 'gitorious',
|
||||
'https://anongit.': ''
|
||||
}
|
||||
def derive_folder_name(url, replaces):
|
||||
sanitize = lambda x: x.replace('/', '.')
|
||||
for service in replaces:
|
||||
if url.startswith(service):
|
||||
@ -43,39 +35,82 @@ def derive_folder_name(url):
|
||||
if url.startswith(generic):
|
||||
url = url[len(generic):]
|
||||
return sanitize(url)
|
||||
raise Exception('malformed url')
|
||||
|
||||
def git_folder_name(url):
|
||||
replaces = {
|
||||
'https://github.com': 'github',
|
||||
'https://git.code.sf.net/p': 'sourceforge',
|
||||
'https://git.tuxfamily.org': 'tuxfamily',
|
||||
'https://git.savannah.gnu.org/git': 'savannah.gnu',
|
||||
'https://gitlab.com': 'gitlab',
|
||||
'https://gitorious.org': 'gitorious',
|
||||
'https://anongit.': '',
|
||||
'https://bitbucket.org': 'bitbucket'
|
||||
}
|
||||
return derive_folder_name(url, replaces)
|
||||
|
||||
|
||||
def clone(url, folder):
|
||||
result = subprocess.run(["git", "clone", url, folder])
|
||||
def git_clone(url, folder):
|
||||
result = subprocess.run(["git", "clone", "--mirror", url, folder])
|
||||
if result.returncode:
|
||||
print(result)
|
||||
|
||||
|
||||
def pull():
|
||||
result = subprocess.run(["git", "pull", "--all"])
|
||||
def git_update(folder):
|
||||
os.chdir(folder)
|
||||
result = subprocess.run(["git", "fetch", "--all"])
|
||||
if result.returncode:
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def svn_folder_name(url):
|
||||
replaces = {
|
||||
'https://svn.code.sf.net/p': 'sourceforge'
|
||||
}
|
||||
return derive_folder_name(url, replaces)
|
||||
|
||||
# get this folder
|
||||
root_folder = os.path.realpath(os.path.dirname(__file__))
|
||||
|
||||
# read archives.json
|
||||
text = read_text(os.path.join(root_folder, 'archives.json'))
|
||||
archives = json.loads(text)
|
||||
print('update {} archives'.format(len(archives)))
|
||||
def svn_clone(url, folder):
|
||||
pass
|
||||
|
||||
# remove those that do not work
|
||||
ignore = ['https://git.code.sf.net/p/dangerdeep/git', 'https://git.code.sf.net/p/erebusrpg/code', 'https://git.code.sf.net/p/rmoffice/code']
|
||||
archives = [x for x in archives if x not in ignore]
|
||||
def svn_update(folder):
|
||||
pass
|
||||
|
||||
|
||||
def hg_folder_name(url):
|
||||
pass
|
||||
|
||||
|
||||
def hg_clone(url, folder):
|
||||
pass
|
||||
|
||||
|
||||
def hg_update(folder):
|
||||
pass
|
||||
|
||||
|
||||
def bzr_folder_name(url):
|
||||
pass
|
||||
|
||||
|
||||
def bzr_clone(url, folder):
|
||||
pass
|
||||
|
||||
|
||||
def bzr_update(folder):
|
||||
pass
|
||||
|
||||
|
||||
def run(type, urls):
|
||||
print('update {} {} archives'.format(len(urls), type))
|
||||
base_folder = os.path.join(root_folder, type)
|
||||
|
||||
# get derived folder names
|
||||
folders = [derive_folder_name(url) for url in archives]
|
||||
folders = [folder_name[type](url) for url in urls]
|
||||
|
||||
# find those folders not used anymore
|
||||
existing_folders = [x for x in os.listdir(root_folder) if os.path.isdir(os.path.join(root_folder, x))]
|
||||
existing_folders = [x for x in os.listdir(base_folder) if os.path.isdir(os.path.join(base_folder, x))]
|
||||
unused_folders = [x for x in existing_folders if x not in folders]
|
||||
print('{} unused archives'.format(len(unused_folders)))
|
||||
if unused_folders:
|
||||
@ -86,19 +121,53 @@ if __name__ == '__main__':
|
||||
print('{} new archives, will clone'.format(len(new_folders)))
|
||||
|
||||
# add root to folders
|
||||
folders = [os.path.join(root_folder, x) for x in folders]
|
||||
os.chdir(root_folder)
|
||||
for folder, archive in zip(folders, archives):
|
||||
folders = [os.path.join(base_folder, x) for x in folders]
|
||||
os.chdir(base_folder)
|
||||
for folder, url in zip(folders, urls):
|
||||
if not os.path.isdir(folder):
|
||||
clone(archive, folder)
|
||||
time.sleep(0.1) # not sure if this is necessary, but there were some issues with too many git operations
|
||||
clone[type](url, folder)
|
||||
|
||||
# at the end update them all
|
||||
for folder in folders:
|
||||
# pull all
|
||||
os.chdir(folder)
|
||||
pull()
|
||||
time.sleep(0.1) # not sure if this is necessary, but there were some issues with too many git operations
|
||||
print('update {}'.format(os.path.basename(folder)))
|
||||
if not os.path.isdir(folder):
|
||||
raise Exception('folder to update not existing')
|
||||
update[type](folder)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
folder_name = {
|
||||
'git': git_folder_name,
|
||||
'svn': svn_folder_name,
|
||||
'hg': hg_folder_name,
|
||||
'bzr': bzr_folder_name
|
||||
}
|
||||
|
||||
clone = {
|
||||
'git': git_clone,
|
||||
'svn': svn_clone,
|
||||
'hg': hg_clone,
|
||||
'bzr': bzr_clone
|
||||
}
|
||||
|
||||
update = {
|
||||
'git': git_update,
|
||||
'svn': svn_update,
|
||||
'hg': hg_update,
|
||||
'bzr': bzr_update
|
||||
}
|
||||
|
||||
# get this folder
|
||||
root_folder = os.path.realpath(os.path.dirname(__file__))
|
||||
|
||||
# read archives.json
|
||||
text = read_text(os.path.join(root_folder, 'archives.json'))
|
||||
archives = json.loads(text)
|
||||
|
||||
for type in archives:
|
||||
urls = archives[type]
|
||||
run(type, urls)
|
||||
|
||||
|
||||
|
||||
@ -605,6 +605,28 @@ def svn_repo(repo):
|
||||
return None
|
||||
|
||||
|
||||
def hg_repo(repo):
|
||||
"""
|
||||
|
||||
"""
|
||||
if repo.startswith('https://bitbucket.org/') and not repo.endswith('.git'):
|
||||
return repo
|
||||
|
||||
if repo.startswith('http://hg.'):
|
||||
return repo
|
||||
|
||||
# not hg
|
||||
return None
|
||||
|
||||
|
||||
def bzr_repo(repo):
|
||||
if repo.startswith('https://code.launchpad.net/'):
|
||||
return repo
|
||||
|
||||
# not bzr
|
||||
return None
|
||||
|
||||
|
||||
def update_primary_code_repositories():
|
||||
|
||||
primary_repos = {'git':[],'svn':[],'hg':[],'bzr':[]}
|
||||
@ -628,10 +650,27 @@ def update_primary_code_repositories():
|
||||
# remove parenthesis and strip of white spaces
|
||||
repo = re.sub(r'\([^)]*\)', '', repo)
|
||||
repo = repo.strip()
|
||||
repo = git_repo(repo)
|
||||
if repo:
|
||||
primary_repos['git'].append(repo)
|
||||
url = git_repo(repo)
|
||||
if url:
|
||||
primary_repos['git'].append(url)
|
||||
consumed = True
|
||||
continue
|
||||
url = svn_repo(repo)
|
||||
if url:
|
||||
primary_repos['svn'].append(url)
|
||||
consumed = True
|
||||
continue
|
||||
url = hg_repo(repo)
|
||||
if url:
|
||||
primary_repos['hg'].append(url)
|
||||
consumed=True
|
||||
continue
|
||||
url = bzr_repo(repo)
|
||||
if url:
|
||||
primary_repos['bzr'].append(url)
|
||||
consumed=True
|
||||
continue
|
||||
|
||||
if not consumed:
|
||||
unconsumed_entries.append([info['title'], info[field]])
|
||||
#if info['code repository']:
|
||||
@ -642,7 +681,7 @@ def update_primary_code_repositories():
|
||||
primary_repos[k] = sorted(set(v))
|
||||
|
||||
# write them to tools/git
|
||||
json_path = os.path.join(games_path, os.path.pardir, 'tools', 'git_archive', 'archives.json')
|
||||
json_path = os.path.join(games_path, os.path.pardir, 'tools', 'archive', 'archives.json')
|
||||
text = json.dumps(primary_repos, indent=1)
|
||||
write_text(json_path, text)
|
||||
|
||||
@ -656,7 +695,7 @@ if __name__ == "__main__":
|
||||
infos = assemble_infos()
|
||||
|
||||
# recount and write to readme
|
||||
update_readme()
|
||||
# update_readme()
|
||||
|
||||
# generate list in toc files
|
||||
update_category_tocs()
|
||||
|
||||
Reference in New Issue
Block a user