""" Runs a series of maintenance operations on the collection of entry files, updating the table of content files for each category as well as creating a statistics file. Counts the number of records each subfolder and updates the overview. Sorts the entries in the contents files of each sub folder alphabetically. This script runs with Python 3, it could also with Python 2 with some minor tweaks probably. """ import os import re import urllib.request import http.client import datetime def get_category_paths(): """ Returns all sub folders of the games path. """ return [os.path.join(games_path, x) for x in os.listdir(games_path) if os.path.isdir(os.path.join(games_path, x))] def get_entry_paths(category_path): """ Returns all files of a category path, except for '_toc.md'. """ return [os.path.join(category_path, x) for x in os.listdir(category_path) if x != '_toc.md' and os.path.isfile(os.path.join(category_path, x))] def read_first_line_from_file(file): """ Convenience function because we only need the first line of a category overview really. """ with open(file, mode='r', encoding='utf-8') as f: line = f.readline() return line def read_interesting_info_from_file(file): """ Parses a file for some interesting fields and concatenates the content. To be displayed after the game name in the category overview. """ with open(file, mode='r', encoding='utf-8') as f: text = f.read() output = [None, None, None] # language regex = re.compile(r"- Language\(s\): (.*)") matches = regex.findall(text) if matches: output[0] = matches[0] # license regex = re.compile(r"- License: (.*)") matches = regex.findall(text) if matches: output[1] = matches[0] # state regex = re.compile(r"- State: (.*)") matches = regex.findall(text) if matches: output[2] = matches[0] output = [x for x in output if x] # eliminate empty entries output = ", ".join(output) return output def update_readme(): """ Recounts entries in sub categories and writes them to the readme. Needs to be performed regularly. """ print('update readme file') # read readme with open(readme_path, mode='r', encoding='utf-8') as f: readme_text = f.read() # compile regex for identifying the building blocks regex = re.compile(r"(# Open Source Games\n\n)(.*)(\nA collection.*)", re.DOTALL) # apply regex matches = regex.findall(readme_text) matches = matches[0] start = matches[0] end = matches[2] # get sub folders category_paths = get_category_paths() # get number of files (minus 1) in each sub folder n = [len(os.listdir(path)) - 1 for path in category_paths] # assemble paths paths = [os.path.join(path, '_toc.md') for path in category_paths] # get titles (discarding first two ("# ") and last ("\n") characters) titles = [read_first_line_from_file(path)[2:-1] for path in paths] # combine titles, category names, numbers in one list info = zip(titles, [os.path.basename(path) for path in category_paths], n) # sort according to sub category title (should be unique) info = sorted(info, key=lambda x:x[0]) # assemble output update = ['- **[{}](games/{}/_toc.md)** ({})\n'.format(*entry) for entry in info] update = "{} entries\n".format(sum(n)) + "".join(update) # insert new text in the middle text = start + "[comment]: # (start of autogenerated content, do not edit)\n" + update + "\n[comment]: # (end of autogenerated content)" + end # write to readme with open(readme_path, mode='w', encoding='utf-8') as f: f.write(text) def update_category_tocs(): """ Lists all entries in all sub folders and generates the list in the toc file. Needs to be performed regularly. """ # get category paths category_paths = get_category_paths() # for each category for category_path in category_paths: print('generate toc for {}'.format(os.path.basename(category_path))) # read toc header line toc_file = os.path.join(category_path, '_toc.md') toc_header = read_first_line_from_file(toc_file) # get paths of all entries in this category entry_paths = get_entry_paths(category_path) # get titles (discarding first two ("# ") and last ("\n") characters) titles = [read_first_line_from_file(path)[2:-1] for path in entry_paths] # get more interesting info more = [read_interesting_info_from_file(path) for path in entry_paths] # combine name and file name info = zip(titles, [os.path.basename(path) for path in entry_paths], more) # sort according to entry title (should be unique) info = sorted(info, key=lambda x:x[0]) # assemble output update = ['- **[{}]({})** ({})\n'.format(*entry) for entry in info] update = "".join(update) # combine toc header text = toc_header + '\n' + "[comment]: # (start of autogenerated content, do not edit)\n" + update + "\n[comment]: # (end of autogenerated content)" # write to toc file with open(toc_file, mode='w', encoding='utf-8') as f: f.write(text) def check_validity_external_links(): """ Checks all external links it can find for validity. Prints those with non OK HTTP responses. Does only need to be run from time to time. """ # regex for finding urls (can be in <> or in () or a whitespace regex = re.compile(r"[\s\n]<(http.+?)>|\]\((http.+?)\)|[\s\n](http[^\s\n]+)") # count number_checked_links = 0 # get category paths category_paths = get_category_paths() # for each category for category_path in category_paths: print('check links for {}'.format(os.path.basename(category_path))) # get entry paths entry_paths = get_entry_paths(category_path) # for each entry for entry_path in entry_paths: # read entry with open(entry_path, 'r', 'utf-8') as f: content = f.read() # apply regex matches = regex.findall(content) # for each match for match in matches: # for each possible clause for url in match: # if there was something if url: try: # without a special headers, frequent 403 responses occur req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}) urllib.request.urlopen(req) except urllib.error.HTTPError as e: print("{}: {} - {}".format(os.path.basename(entry_path), url, e.code)) except http.client.RemoteDisconnected: print("{}: {} - disconnected without response".format(os.path.basename(entry_path), url)) number_checked_links += 1 if number_checked_links % 50 == 0: print("{} links checked".format(number_checked_links)) print("{} links checked".format(number_checked_links)) def fix_notation(): """ Changes notation, quite special. Only run when needed. """ regex = re.compile(r"- License details:(.*)") # get category paths category_paths = get_category_paths() # for each category for category_path in category_paths: # get paths of all entries in this category entry_paths = get_entry_paths(category_path) for entry_path in entry_paths: # read it line by line with open(entry_path, 'r', 'utf-8') as f: content = f.readlines() # apply regex on every line matched_lines = [regex.findall(line) for line in content] # loop over all the lines for line, match in enumerate(matched_lines): if match: match = match[0] # patch content content[line] = "- Code license details:{}\n".format(match) # write it line by line with open(entry_path, "w", 'utf-8') as f: f.writelines(content) def regular_replacements(): """ Replacing some stuff by shortcuts. Can be run regularly """ # get category paths category_paths = get_category_paths() # for each category for category_path in category_paths: # get paths of all entries in this category entry_paths = get_entry_paths(category_path) for entry_path in entry_paths: # read it line by line with open(entry_path, 'r', 'utf-8') as f: content = f.read() # now the replacements content = content.replace('?source=navbar', '') # sourceforge specific content = content.replace('single player', 'SP') content = content.replace('multi player', 'MP') # write it line by line with open(entry_path, "w", 'utf-8') as f: f.write(content) def check_template_leftovers(): """ Checks for template leftovers. """ check_strings = ['# {NAME}', '_{One line description}_', '- Home: {URL}', '- Media: {URL}', '- Download: {URL}', '- State: beta, mature, inactive since', '- Keywords: SP, MP, RTS, TBS (if none, remove the line)', '- Code: primary repository (type if not git), other repositories (type if not git)', '- Language(s): {XX}', '- License: {XX} (if special, include link)', '{XXX}'] # get category paths category_paths = get_category_paths() # for each category for category_path in category_paths: # get paths of all entries in this category entry_paths = get_entry_paths(category_path) for entry_path in entry_paths: # read it line by line with open(entry_path, 'r', 'utf-8') as f: content = f.read() for check_string in check_strings: if content.find(check_string) >= 0: print('{}: found {}'.format(os.path.basename(entry_path), check_string)) def parse_entry(content): """ Returns a dictionary of the features of the content """ info = {} # state regex = re.compile(r"- State: (.*)") matches = regex.findall(content) if matches: # first remove everything in parenthesis states = re.sub(r'\([^)]*\)', '', matches[0]) states = states.split(',') states = [x.strip() for x in states] if 'beta' in states: info['state'] = 'beta' elif 'mature' in states: info['state'] = 'mature' else: print('Neither beta nor mature in state tag: {}'.format(content)) inactive = next((int(x[14:]) for x in states if x.startswith('inactive since')), None) # only the year if inactive: info['inactive'] = inactive # language regex = re.compile(r"- Language\(s\): (.*)") matches = regex.findall(content) if matches: # first remove everything in parenthesis languages = re.sub(r'\([^)]*\)', '', matches[0]) languages = languages.split(',') languages = [x.strip() for x in languages] info['language'] = languages # license regex = re.compile(r"- Code license: (.*)") matches = regex.findall(content) if matches: # first remove everything in parenthesis license = re.sub(r'\([^)]*\)', '', matches[0]) info['license'] = license return info def generate_statistics(): """ """ statistics_path = os.path.join(games_path, 'statistics.md') statistics = '[comment]: # (autogenerated content, do not edit)\n# Statistics\n\n' # get category paths category_paths = get_category_paths() # for each category infos = [] for category_path in category_paths: # get paths of all entries in this category entry_paths = get_entry_paths(category_path) for entry_path in entry_paths: # read it line by line with open(entry_path, mode='r', encoding='utf-8') as f: content = f.read() info = parse_entry(content) info['file'] = os.path.basename(entry_path)[:-3] # [:-3] to cut off the .md infos.append(info) # total number number_entries = len(infos) rel = lambda x: x / number_entries * 100 # converion to percent statistics += 'analyzed {} entries on {}\n\n'.format(number_entries, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) # State (beta, mature, inactive) statistics += '## State\n\n' number_state_beta = sum(1 for x in infos if 'state' in x and x['state'] == 'beta') number_state_mature = sum(1 for x in infos if 'state' in x and x['state'] == 'mature') number_inactive = sum(1 for x in infos if 'inactive' in x) statistics += '- mature: {} ({:.1f}%)\n- beta: {} ({:.1f}%)\n- inactive: {} ({:.1f}%)\n\n'.format(number_state_mature, rel(number_state_mature), number_state_beta, rel(number_state_beta), number_inactive, rel(number_inactive)) if number_inactive > 0: entries_inactive = [(x['file'], x['inactive']) for x in infos if 'inactive' in x] entries_inactive.sort(key=lambda x: x[0]) # first sort by name entries_inactive.sort(key=lambda x: -x[1]) # then sort by inactive year (more recently first) entries_inactive = ['{} ({})'.format(*x) for x in entries_inactive] statistics += '##### Inactive State\n\n' + ', '.join(entries_inactive) + '\n\n' entries_no_state = [x['file'] for x in infos if 'state' not in x] if entries_no_state: entries_no_state.sort() statistics += '##### Without state tag ({})\n\n'.format(len(entries_no_state)) + ', '.join(entries_no_state) + '\n\n' # Language statistics += '## Languages\n\n' number_no_language = sum(1 for x in infos if 'language' not in x) if number_no_language > 0: statistics += 'Without language tag: {} ({:.1f}%)\n\n'.format(number_no_language, rel(number_no_language)) entries_no_language = [x['file'] for x in infos if 'language' not in x] entries_no_language.sort() statistics += ', '.join(entries_no_language) + '\n\n' # get all languages together languages = [] for info in infos: if 'language' in info: languages.extend(info['language']) unique_languages = set(languages) unique_languages = [(l, languages.count(l) / len(languages)) for l in unique_languages] unique_languages.sort(key=lambda x: x[0]) # first sort by name unique_languages.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first) unique_languages = ['- {} ({:.1f}%)\n'.format(x[0], x[1]*100) for x in unique_languages] statistics += '##### Language frequency\n\n' + ''.join(unique_languages) + '\n' # Licenses statistics += '## Code licenses\n\n' number_no_license = sum(1 for x in infos if 'license' not in x) if number_no_license > 0: statistics += 'Without license tag: {} ({:.1f}%)\n\n'.format(number_no_license, rel(number_no_license)) entries_no_license = [x['file'] for x in infos if 'license' not in x] entries_no_license.sort() statistics += ', '.join(entries_no_license) + '\n\n' # get all licenses together licenses = [] for info in infos: if 'license' in info: licenses.append(info['license']) unique_licenses = set(licenses) unique_licenses = [(l, licenses.count(l) / len(licenses)) for l in unique_licenses] unique_licenses.sort(key=lambda x: x[0]) # first sort by name unique_licenses.sort(key=lambda x: -x[1]) # then sort by occurrence (highest occurrence first) unique_licenses = ['- {} ({:.1f}%)\n'.format(x[0], x[1]*100) for x in unique_licenses] statistics += '##### Licenses frequency\n\n' + ''.join(unique_licenses) + '\n' with open(statistics_path, mode='w', encoding='utf-8') as f: f.write(statistics) if __name__ == "__main__": # paths games_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'games')) readme_path = os.path.join(games_path, os.pardir, 'README.md') # recount and write to readme update_readme() # generate list in toc files update_category_tocs() # generate report generate_statistics() # check for unfilled template lines # check_template_leftovers() # check external links (only rarely) #check_validity_external_links() # special, only run when needed # fix_notation() # regular replacements #regular_replacements()