AnotherFoxGuy/opensourcegames
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

new entries, svn to git manual conversion of holy spirit

Browse Source
This commit is contained in:
Trilarion
2018-07-11 11:16:07 +02:00
parent 316e23ab17
commit d9754b45ff
14 changed files with 261 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tools/archives.json
View File

@ -27,6 +27,7 @@
"https://git.code.sf.net/p/freelords/git",
"https://git.code.sf.net/p/gigalomania/code",
"https://git.code.sf.net/p/lipsofsuna/code",
"https://git.code.sf.net/p/monstergenerato/code",
"https://git.code.sf.net/p/mpango/git",
"https://git.code.sf.net/p/nazghul/git",
"https://git.code.sf.net/p/rmoffice/code",
@ -122,6 +123,7 @@
"https://github.com/Trilarion/dark-oberon.git",
"https://github.com/Trilarion/deity.git",
"https://github.com/Trilarion/dungeonmap.git",
"https://github.com/Trilarion/endlessdungeons.git",
"https://github.com/Trilarion/eos-game.git",
"https://github.com/Trilarion/freetrain.git",
"https://github.com/Trilarion/glest.git",
@ -328,6 +330,7 @@
"https://gitlab.com/Trilarion/janag.git",
"https://gitlab.com/Trilarion/koboldsquest2.git",
"https://gitlab.com/Trilarion/kursk.git",
"https://gitlab.com/Trilarion/lechemindeladam.git",
"https://gitlab.com/Trilarion/lincity.git",
"https://gitlab.com/Trilarion/machinations.git",
"https://gitlab.com/Trilarion/mercenarycommander.git",
@ -380,7 +383,6 @@
"https://svn.code.sf.net/p/freesynd/code/",
"https://svn.code.sf.net/p/fsc/code/",
"https://svn.code.sf.net/p/kralovstvi/code/",
"https://svn.code.sf.net/p/lechemindeladam/code/",
"https://svn.code.sf.net/p/low/code/",
"https://svn.code.sf.net/p/opencity/code/",
"https://svn.code.sf.net/p/opengeneral/code/",
@ -406,7 +408,8 @@
"https://bitbucket.org/giszmo/glob2",
"https://bitbucket.org/mstrobel/supremacy",
"https://bitbucket.org/rude/love",
"https://bitbucket.org/sumwars/sumwars-code/src"
"https://bitbucket.org/sumwars/sumwars-code/src",
"https://bitbucket.org/thesheep/fujo/src"
],
"bzr": [
"https://code.launchpad.net/heroesofwesnoth",

123
tools/lechemindeladam_svn_to_git.py
View File

@ -3,7 +3,7 @@ The svn is too big to be automatically imported to git (and Github) because ther
Needs a manual solution.
TODO use git lfs migrate later on the elements
TODO check for sufficient disc space before checkout
TODO instead of svn export for every revision, checkout and then update to revision (reduced bandwith)
"""
import json
@ -36,6 +36,13 @@ def special_treatment(destination, revision):
"""
# copy content of trunk to base
if 2270 <= revision <= 2420:
source = os.path.join(destination, 'trunk')
if os.path.isdir(source):
copy_tree(source, destination)
shutil.rmtree(source)
# copy all important files from Holyspirit/Holyspirit and delete it
if 5 <= revision <= 330:
source = os.path.join(destination, 'Holyspirit', 'Holyspirit')
@ -49,31 +56,50 @@ def special_treatment(destination, revision):
shutil.rmtree(os.path.join(destination, 'Holyspirit'))
# copy all important files from Holyspirit and delete it
if 337 <= revision <= 1700:
if 337 <= revision <= 2268:
source = os.path.join(destination, 'Holyspirit')
if os.path.isdir(source):
data = os.path.join(source, 'Data')
if os.path.isdir(data):
# shutil.copytree(data, os.path.join(destination, 'Data'))
shutil.move(data, destination)
files = [x for x in os.listdir(source) if x.endswith('.txt') or x.endswith('.conf')]
target = os.path.join(destination, 'Meta')
if not os.path.isdir(target):
os.mkdir(target)
files = [x for x in os.listdir(source) if x.endswith('.txt') or x.endswith('.conf') or x.endswith('.ini')]
for file in files:
shutil.move(os.path.join(source, file), destination)
shutil.move(os.path.join(source, file), target)
# remove it
shutil.rmtree(source)
# copy data folder vom HolySpiritJE and delete it
if 2012 <= revision <= 2269:
source = os.path.join(destination, 'HolyspiritJE')
if os.path.isdir(source):
data = os.path.join(source, 'Data')
if os.path.isdir(data):
shutil.move(data, os.path.join(destination, 'DataJE'))
target = os.path.join(destination, 'MetaJE')
if not os.path.isdir(target):
os.mkdir(target)
files = [x for x in os.listdir(source) if x.endswith('.txt') or x.endswith('.conf') or x.endswith('.ini')]
for file in files:
shutil.move(os.path.join(source, file), target)
# remove it
shutil.rmtree(source)
# remove Holyspirit3 folder
if 464 <= revision <= 1700:
if 464 <= revision <= 2268:
remove_folders(destination, 'Holyspirit3')
# remove Holyspirit2 folder
if 659 <= revision <= 1700:
if 659 <= revision <= 2268:
remove_folders(destination, 'Holyspirit2')
# remove Launcher/release
if 413 <= revision <= 1700:
if 413 <= revision <= 2420:
source = os.path.join(destination, 'Launcher')
remove_folders(source, ('debug', 'release'))
remove_folders(source, ('bin', 'debug', 'release', 'obj'))
# delete all *.dll, *.exe in base folder
if 3 <= revision <= 9:
@ -87,11 +113,14 @@ def special_treatment(destination, revision):
remove_folders(destination, 'Cross')
# delete personal photos
if 374 <= revision <= 1700:
if 374 <= revision <= 2267:
remove_folders(destination, 'Photos')
if 2268 <= revision <= 2420:
source = os.path.join(destination, 'Media')
remove_folders(source, 'Photos')
# move empire of steam out
if 1173 <= revision <= 1700:
if 1173 <= revision <= 2420:
folder = os.path.join(destination, 'EmpireOfSteam')
if os.path.isdir(folder):
# move to empire path
@ -99,27 +128,54 @@ def special_treatment(destination, revision):
shutil.move(folder, empire)
# holy editor cleanup
if 1078 <= revision <= 1700:
if 1078 <= revision <= 2420:
source = os.path.join(destination, 'HolyEditor')
remove_folders(source, ('bin', 'release', 'debug', 'obj'))
remove_files(source, 'moc.exe')
# source folder cleanup
if 939 <= revision <= 1700:
if 939 <= revision <= 2420:
source = os.path.join(destination, 'Source')
remove_folders(source, 'HS')
remove_files(source, 'HS.zip')
# sourceM folder cleanup
if 2110 <= revision <= 2270:
source = os.path.join(destination, 'SourceM')
remove_folders(source, 'HS')
# sourceNewApi cleanup
if 2261 <= revision <= 2269:
source = os.path.join(destination, 'SourceNewApi')
remove_folders(source, 'HS')
# Autres folder cleanup
if 1272 <= revision <= 1700:
if 1272 <= revision <= 2267:
source = os.path.join(destination, 'Autres')
remove_folders(source, ('conf', 'db', 'hooks', 'locks'))
remove_files(source, ('format', 'maj.php'))
# Media/Other folder cleanup
if 2268 <= revision <= 2420:
source = os.path.join(destination, 'Media', 'Other')
remove_files(source, ('format', 'maj.php'))
# remove Holyspirit-Demo
if 1668 <= revision <= 1700:
if 1668 <= revision <= 2268:
remove_folders(destination, 'Holyspirit_Demo')
# remove Debug.rar
if 1950 <= revision <= 2420:
remove_files(destination, 'Debug.rar')
# remove 3dparty folder
if 2273 <= revision <= 2420:
remove_folders(destination, '3dparty')
# branches cleanup
if 2270 <= revision <= 2420:
remove_folders(destination, 'branches')
def delete_global_excludes(folder):
"""
@ -161,10 +217,13 @@ def list_large_unwanted_files(folder):
return output
def checkout(revision_start, revision_end):
def checkout(revision_start, revision_end=None):
"""
"""
if not revision_end:
revision_end = revision_start
assert revision_end >= revision_start
for revision in range(revision_start, revision_end + 1):
@ -182,7 +241,16 @@ def checkout(revision_start, revision_end):
# checkout
start_time = time.time()
subprocess_run(['svn', 'export', '-r{}'.format(revision), svn_url, destination])
# sometimes checkout fails for reasons like "svn: E000024: Can't open file '/svn/p/lechemindeladam/code/db/revs/1865': Too many open files", we try again and again in these cases
while True:
try:
subprocess_run(['svn', 'export', '-r{}'.format(revision), svn_url, destination])
break
except:
print('problem with export, will try again')
if os.path.isdir(destination):
shutil.rmtree(destination)
print('checkout took {:.1f}s'.format(time.time() - start_time))
@ -410,4 +478,25 @@ if __name__ == "__main__":
# fix_revision(1471, 1700)
# gitify(1471, 1700)
checkout(1701, 2100)
# checkout(1701, 1900)
# fix_revision(1701, 1900)
# gitify(1701, 1900)
# checkout(1901, 2140)
# fix_revision(1901, 2140)
# gitify(1901, 2140)
# checkout(2141, 2388)
# fix_revision(2141, 2388)
# gitify(2141, 2388)
# checkout(2389, 2420)
# fix_revision(2389, 2420)
# gitify(2389, 2420)
# run the following commands in the git bash
# git config credential.useHttpPath true
# git lfs install
# git lfs migrate import --include-ref=master --include="Zombie_paysan.rs.hs,Witch_monster.rs.hs,WanderingStones.rs.hs,TwoWeapons.rs.hs,TwoHands.rs.hs,TwoHand.rs.hs,Reaper.rs.hs,Peasant_crossbow.rs.hs,Peasant_club.rs.hs,OneHand.rs.hs,Offspring_champion.rs.hs,Mimic.rs.hs,LordSkeleton.rs.hs,Goule.rs.hs,ErrantRoche.rs.hs,DemonicPriest0.rs.hs,DemonicPriest.rs.hs,Brute.rs.hs,20575__dobroide__20060706.night.forest02.wav,31464__offtheline__Morning_Sounds.wav,47989__Luftrum__forestsurroundings.wav,ambiance.wav,Catacombs0.wav,Pluie.wav,Taverne fusion.png,Abbey.ogg,AgrarianLands0.ogg,AgrarianLands1.ogg,Boss0.ogg,Catacombs0.ogg,Catacombs1.ogg,DarkForest.ogg,Forest_ambient0.ogg,Johannes.ogg,OWC.ogg"
# then add remote and push (done)

40
tools/maintenance.py
View File

@ -470,8 +470,39 @@ def generate_statistics():
unique_keywords = [(l, keywords.count(l) / len(keywords)) for l in unique_keywords]
unique_keywords.sort(key=lambda x: x[0]) # first sort by name
unique_keywords.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first)
unique_keywords = ['- {} ({:.1f}%)\n'.format(x[0], x[1]*100) for x in unique_keywords]
statistics += '##### Keywords frequency\n\n' + ''.join(unique_keywords) + '\n'
unique_keywords = ['- {} ({:.1f}%)'.format(x[0], x[1]*100) for x in unique_keywords]
statistics += '##### Keywords frequency\n\n' + '\n'.join(unique_keywords) + '\n\n'
# no download or play field
statistics += '## Entries without download or play\n\n'
entries = []
for info in infois:
if 'download' not in info and 'play' not in info:
entries.append(info['title'])
entries.sort()
statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
# code hosted not on github, gitlab, bitbucket, launchpad, sourceforge
popular_code_repositories = ('github.com', 'gitlab.com', 'bitbucket.org', 'code.sf.net', 'code.launchpad.net')
statistics += '## Entries with a code repository but not on a popular site\n\n'
entries = []
field = 'code repository'
for info in infois:
if info[field]:
popular = False
for repo in info[field]:
for popular_repo in popular_code_repositories:
if popular_repo in repo:
popular = True
break
# if there were repositories, but none popular, add them to the list
if not popular:
entries.append(info['title'])
# print(info[field])
entries.sort()
statistics += '{}: '.format(len(entries)) + ', '.join(entries) + '\n\n'
with open(statistics_path, mode='w', encoding='utf-8') as f:
f.write(statistics)
@ -654,8 +685,9 @@ def update_primary_code_repositories():
if not consumed:
unconsumed_entries.append([info['title'], info[field]])
if info['code repository']:
print('Entry "{}" unconsumed repo: {}'.format(info['title'], info[field]))
# print output
#if info['code repository']:
# print('Entry "{}" unconsumed repo: {}'.format(info['title'], info[field]))
#if not info['code repository']:
# print('Entry "{}" unconsumed repo: {}'.format(info['title'], info[field]))

4
tools/utils/utils.py
View File

@ -5,7 +5,6 @@ Utilities for the tools.
import os
import shutil
import subprocess
import sys
import tarfile
import time
import urllib.request
@ -156,12 +155,13 @@ def subprocess_run(cmd, display=True):
print("error {} in call {}".format(result.returncode, cmd))
print(result.stdout.decode('cp1252'))
print(result.stderr.decode('cp1252'))
sys.exit(-1)
raise RuntimeError()
if display:
print(' output: {}'.format(result.stdout.decode('cp1252')))
return result.stdout.decode('cp1252')
# TODO need move_tree
def copy_tree(source, destination):
"""
Copies the full content of one directory into another avoiding the use of distutils.di_util.copy_tree because that