maintenance of entries, backlog, ..

code/utils/constants.py

@@ -14,6 +14,9 @@ code_path = os.path.join(root_path, 'code')
 inspirations_file = os.path.join(root_path, 'inspirations.md')
 developer_file = os.path.join(root_path, 'developers.md')
 
+backlog_file = os.path.join(code_path, 'backlog.txt')
+rejected_file = os.path.join(code_path, 'rejected.txt')
+
 # local config
 local_config_file = os.path.join(root_path, 'local-config.ini')
 
@@ -43,7 +46,8 @@ valid_fields = ('File', 'Title') + valid_properties + ('Note', 'Building')
 
 url_fields = ('Home', 'Media', 'Play', 'Download', 'Code repository')
 
-valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://', '@see-', '@not-', '?')
+valid_url_prefixes = ('http://', 'https://', 'git://', 'svn://', 'ftp://', 'bzr://')
+extended_valid_url_prefixes = valid_url_prefixes + ('@see-', '@not-', '?')
 
 valid_building_properties = ('Build system', 'Build instructions')
 valid_building_fields = valid_building_properties + ('Note',)

code/utils/osg.py

@@ -321,7 +321,7 @@ def check_and_process_entry(entry):
         for value in values:
             if value.value.startswith('<') and value.value.endswith('>'):
                 value.value = value.value[1:-1]
-            if not any(value.startswith(x) for x in valid_url_prefixes):
+            if not any(value.startswith(x) for x in extended_valid_url_prefixes):
                 message += 'URL "{}" in field "{}" does not start with a valid prefix'.format(value, field)
 
     # github/gitlab repositories should end on .git and should start with https
@@ -381,7 +381,6 @@ def extract_inactive_year(entry):
     else:
         return None
 
-
 def write_entries(entries):
     """
 
@@ -467,3 +466,37 @@ def create_entry_content(entry):
         content += entry['Building']['Note']
 
     return content
+
+
+def is_url(str):
+    """
+    Could be too generous. See https://stackoverflow.com/questions/7160737/how-to-validate-a-url-in-python-malformed-or-not for other possibilities.
+    :param str:
+    :return:
+    """
+    if any(str.startswith(x) for x in valid_url_prefixes) and not ' ' in str:
+        return True
+    return False
+
+
+def all_urls(entries):
+    """
+    Gets all urls of all entries in a dictionary (key=url value=list of entries (file name) with this url
+    :param entries: 
+    :return: 
+    """
+    urls = {}
+    # iterate over entries
+    for entry in entries:
+        file = entry['File']
+        for field in url_fields:  # TODO there are other fields, maybe just regex on the whole content
+            for value in entry.get(field, []):
+                if value.comment:
+                    value = value.value + ' ' + value.comment
+                else:
+                    value = value.value
+                for subvalue in value.split(' '):
+                    subvalue = subvalue.strip()
+                    if is_url(subvalue):
+                        urls[subvalue] = urls.get(subvalue, []) + [file]
+    return urls