GmCapsule [gsorg-style]

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..db85930
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+__pycache__
+.certs/
+localhost/
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f7c555b
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# GmCapsule: Extensible Server for Gemini and Titan
+
diff --git a/gemini.py b/gemini.py
new file mode 100644
index 0000000..7297a3b
--- /dev/null
+++ b/gemini.py
@@ -0,0 +1,296 @@
+# Copyright (c) 2021-2022 Jaakko Keränen <jaakko.keranen@iki.fi>
+# License: BSD 2-Clause
+
+import fnmatch
+import hashlib
+import socket
+import time
+from urllib.parse import urlparse
+
+import OpenSSL.crypto
+from OpenSSL import SSL, crypto
+
+
+def report_error(stream, code, msg):
+    stream.sendall(f'{code} {msg}\r\n'.encode('utf-8'))
+
+
+class Identity:
+    def __init__(self, cert):
+        self.cert = cert
+        m = hashlib.sha256()
+        m.update(crypto.dump_certificate(crypto.FILETYPE_ASN1, self.cert))
+        self.fp_cert = m.hexdigest()
+        self.pubkey = self.cert.get_pubkey()
+        m = hashlib.sha256()
+        m.update(crypto.dump_publickey(crypto.FILETYPE_ASN1, self.pubkey))
+        self.fp_pubkey = m.hexdigest()
+
+    def __str__(self):
+        return f"{self.fp_cert};{self.fp_pubkey}"
+
+    def subject(self):
+        return ''.join('/{:s}={:s}'.format(name.decode(), value.decode())
+            for name, value in self.cert.get_subject().get_components())
+
+
+class Request:
+    def __init__(self, identity=None, scheme='gemini', hostname='', path='', query='',
+                 remote_address=None, content_token=None, content_mime=None, content=None):
+        self.remote_address = remote_address
+        self.scheme = scheme
+        self.identity = identity
+        self.hostname = hostname
+        self.path = path
+        self.query = query
+        self.content_token = content_token
+        self.content_mime = content_mime
+        self.content = content
+
+
+def verify_callback(connection, cert, err_num, err_depth, ret_code):
+    #print("verify_callback:", connection, cert, ret_code)
+    return True
+
+
+class Cache:
+    def __init__(self):
+        pass
+
+    def try_load(self, path):
+        """Load the contents of `path` from cache (media type, content)."""
+        return None, None
+
+    def save(self, path, media_type, content):
+        return
+
+
+class Server:
+    def __init__(self, hostname_or_hostnames, cert_path, key_path,
+                 address='localhost', port=1965,
+                 cache=None, session_id=None, max_upload_size=0):
+        self.hostnames = [hostname_or_hostnames] \
+            if type(hostname_or_hostnames) == str else hostname_or_hostnames
+        self.address = address
+        self.port = port
+        self.entrypoints = {'gemini': {}, 'titan': {}}
+        for proto in ['gemini', 'titan']:
+            self.entrypoints[proto] = {}
+            for hostname in self.hostnames:
+                self.entrypoints[proto][hostname] = []
+        self.cache = cache
+        self.max_upload_size = max_upload_size
+
+        self.context = SSL.Context(SSL.TLS_SERVER_METHOD)
+        self.context.use_certificate_file(str(cert_path))
+        self.context.use_privatekey_file(str(key_path))
+        self.context.set_verify(SSL.VERIFY_PEER, verify_callback)
+        if session_id:
+            if type(session_id) != bytes:
+                raise Exception("session_id type must be `bytes`")
+            self.context.set_session_id(session_id)
+
+        attempts = 60
+        print(f'Opening port {port}...')
+        while True:
+            try:
+                self.sock = socket.socket()
+                self.sock.bind((address, port))
+                self.sock.listen(5)
+                self.sv_conn = SSL.Connection(self.context, self.sock)
+                self.sv_conn.set_accept_state()
+                break
+            except:
+                attempts -= 1
+                if attempts == 0:
+                    raise Exception(f'Failed to open port {port} for listening')
+                time.sleep(2.0)
+                print('...')
+        print(f'Server started on port {port}')
+
+    def add_entrypoint(self, protocol, hostname, path_pattern, entrypoint):
+        self.entrypoints[protocol][hostname].append((path_pattern, entrypoint))
+
+    def __setitem__(self, key, value):
+        #if key.endswith('*'):
+        #    self.wild_entrypoints[key[:-1]] = value
+        #else:
+        #    self.entrypoints[key] = value
+        for hostname in self.hostnames:
+            self.add_entrypoint('gemini', hostname, key, value)
+
+    # def __getitem__(self, key):
+    #     if key.endswith('*'):
+    #         return self.wild_entrypoints[key[:-1]]
+    #     return self.entrypoints[key]
+
+    def run(self):
+        while True:
+            try:
+                stream = None
+                try:
+                    stream, from_addr = self.sv_conn.accept()
+                    #print(stream, from_addr)
+                    self.process_request(stream, from_addr)
+                except Exception as ex:
+                    import traceback
+                    traceback.print_exc()
+                    print(ex)
+                    if stream:
+                        report_error(stream, 42, str(ex))
+                finally:
+                    if stream:
+                        stream.shutdown()
+                        #print('Goodbye', from_addr)
+            except Exception as ex:
+                print(ex)
+
+    def find_entrypoint(self, protocol, hostname, path):
+        try:
+            for entry in self.entrypoints[protocol][hostname]:
+                if len(entry[0]) == 0 or fnmatch.fnmatch(path, entry[0]):
+                    return entry[1]
+        except:
+            return None
+
+        # # Check the more specific virtual host entrypoint first.
+        # virt_path = f":{hostname}:{path}"
+        # if virt_path in self.entrypoints:
+        #     return self.entrypoints[virt_path]
+        # for entry in self.wild_entrypoints:
+        #     if virt_path.startswith(entry):
+        #         return self.wild_entrypoints[entry]
+
+        # if path in self.entrypoints:
+        #     return self.entrypoints[path]
+        # for entry in self.wild_entrypoints:
+        #     if path.startswith(entry):
+        #         return self.wild_entrypoints[entry]
+
+        return None
+
+    def process_request(self, stream, from_addr):
+        print(time.strftime('%Y-%m-%d %H:%M:%S'))
+        data = bytes()
+        MAX_LEN = 1024
+        request = None
+        expected_size = 0
+        req_token = None
+        req_mime = None
+        incoming = stream.recv(MAX_LEN)
+
+        print(dir(stream))
+
+        while len(data) < MAX_LEN:
+            data += incoming
+            crlf_pos = data.find(b'\r\n')
+            if crlf_pos >= 0:
+                request = data[:crlf_pos].decode('utf-8')
+                data = data[crlf_pos + 2:]
+                break
+            if len(data) == MAX_LEN:
+                report_error(stream, 59, 'Request is too long')
+                return
+            incoming = stream.recv(MAX_LEN - len(data))
+            if len(incoming) == 0:
+                break
+
+        if not request or not (request.startswith('gemini:') or request.startswith('titan:')):
+            report_error(stream, 59, "Unsupported protocol")
+            return
+
+        if request.startswith('titan:'):
+            # Read the rest of the data.
+            parms = request.split(';')
+            request = parms[0]
+            for p in parms:
+                if p.startswith('size='):
+                    expected_size = int(p[5:])
+                elif p.startswith('token='):
+                    req_token = p[6:]
+                elif p.startswith('mime='):
+                    req_mime = p[5:]
+            if expected_size > self.max_upload_size and self.max_upload_size > 0:
+                report_error(stream, 59, "Maximum content length exceeded")
+                return
+            while len(data) < expected_size:
+                incoming = stream.recv(65536)
+                if len(incoming) == 0:
+                    break
+                data += incoming
+            if len(data) != expected_size:
+                report_error(stream, 59, "Invalid content length")
+                return
+        else:
+            # No Payload in Gemini.
+            if len(data):
+                report_error(stream, 59, "Gemini disallows request content")
+                return
+
+        url = urlparse(request)
+        cl_cert = stream.get_peer_certificate()
+        identity = Identity(cl_cert) if cl_cert else None
+        path = url.path
+        if path == '':
+            path = '/'
+        # TODO: get TLS SNI
+        hostname = url.hostname
+        entrypoint = self.find_entrypoint(url.scheme, hostname, path)
+        print(entrypoint)
+        cache = None if identity or len(url.query) > 0 else self.cache
+        is_from_cache = False
+
+        # print(f'Request : {request}')
+        # print(f'Cert    : {cl_cert}')
+
+        if entrypoint:
+            # Check the cache first.
+            if cache:
+                media, content = cache.try_load(path)
+                if not media is None:
+                    response = 20, media, content
+                    is_from_cache = True
+                    print('%d bytes from cache, %s' % (len(content), media))
+
+            # Process the request normally if there is nothing cached.
+            if not is_from_cache:
+                response = entrypoint(Request(
+                    identity,
+                    remote_address=from_addr,
+                    scheme=url.scheme,
+                    hostname=hostname,
+                    path=path,
+                    query=url.query,
+                    content_token=req_token,
+                    content_mime=req_mime,
+                    content=data if len(data) else None
+                ))
+
+            # Determine status code, meta line, and body content.
+            if type(response) == tuple:
+                if len(response) == 2:
+                    status = response[0]
+                    meta = response[1]
+                    response = ''
+                else:
+                    status = response[0]
+                    meta = response[1]
+                    response = response[2]
+            else:
+                status = 20
+                meta = 'text/gemini;charset=utf-8'
+
+            if response == None:
+                response_data = b''
+            elif type(response) == str:
+                response_data = response.encode('utf-8')
+            else:
+                response_data = response
+
+            stream.sendall(f'{status} {meta}\r\n'.encode('utf-8') + response_data)
+
+            # Save to cache.
+            if not is_from_cache and cache and status == 20:
+                cache.save(path, meta, response_data)
+        else:
+            report_error(stream, 50, 'Permanent failure')
diff --git a/gmcapsule.py b/gmcapsule.py
new file mode 100644
index 0000000..4886141
--- /dev/null
+++ b/gmcapsule.py
@@ -0,0 +1,109 @@
+# Copyright (c) 2022 Jaakko Keränen <jaakko.keranen@iki.fi>
+# License: BSD 2-Clause
+
+import configparser
+import fnmatch
+import importlib
+import mimetypes
+import os
+import subprocess
+from pathlib import Path
+
+import gemini
+
+
+class Config:
+    def __init__(self):
+        # TODO: Get this using configparser
+        self.hostnames = ['localhost']
+        self.address = '0.0.0.0'
+        self.port = 1965
+        self.certs_dir = Path('.certs')
+        self.root_dir = Path('.')        # vhosts as subdirs
+        self.mod_dir = Path('modules')   # extension modules
+        self.max_upload_size = 10 * 1024 * 1024
+        self.cgi = {
+            'gemini': {
+                'localhost': [
+                    ('/test', ['/bin/ls', '-l'])
+                ]
+            },
+            'titan': {
+                'localhost': [
+                    ('/test', ['printenv']),
+                    ('/test/*', ['printenv'])
+                ]
+            }
+        }
+
+
+class Capsule:
+    _capsule = None
+
+    def __init__(self, cfg):
+        Capsule._capsule = self
+        self.cfg = cfg
+        self.sv = gemini.Server(
+            cfg.hostnames,
+            cfg.certs_dir / 'cert.pem',
+            cfg.certs_dir / 'key.pem',
+            address=cfg.address,
+            port=cfg.port,
+            session_id=f'GmCapsule:{cfg.port}'.encode('utf-8'),
+            max_upload_size=cfg.max_upload_size
+        )
+        # Modules define the entrypoints.
+        self.load_modules()
+
+    @staticmethod
+    def config():
+        return Capsule._capsule.cfg
+
+    def add(self, path, entrypoint, hostname=None, protocol='gemini'):
+        if hostname:
+            self.sv.add_entrypoint(protocol, hostname, path, entrypoint)
+        else:
+            for hostname in self.cfg.hostnames:
+                if not hostname:
+                    raise Exception(f'invalid hostname: "{hostname}"')
+                self.sv.add_entrypoint(protocol, hostname, path, entrypoint)
+
+    def load_modules(self):
+        for mod_file in sorted(os.listdir(self.cfg.mod_dir)):
+            if mod_file.endswith('.py'):
+                path = (self.cfg.mod_dir / mod_file).resolve()
+                name = mod_file[:-3]
+                print('Module:', name)
+                loader = importlib.machinery.SourceFileLoader(name, str(path))
+                spec = importlib.util.spec_from_loader(name, loader)
+                mod = importlib.util.module_from_spec(spec)
+                loader.exec_module(mod)
+                mod.init(self)
+
+    def run(self):
+        self.sv.run()
+
+
+def get_mime_type(path):
+    p = str(path)
+    lp = p.lower()
+    if lp.endswith('.txt'):
+        return 'text/plain'
+    if lp.endswith('.gmi') or lp.endswith('.gemini'):
+        return 'text/gemini'
+    if lp.endswith('.md') or lp.endswith('.markdown') or lp.endswith('.mdown'):
+        return 'text/markdown'
+
+    if not path.exists():
+        return None
+
+    mt = mimetypes.guess_type(path)[0]
+    if mt is not None:
+       return mt
+
+    try:
+        return subprocess.check_output([
+            '/usr/bin/env', 'file', '--mime-type', '-b', p
+            ]).decode('utf-8').strip()
+    except:
+        return 'application/octet-stream'
diff --git a/gmcapsuled b/gmcapsuled
new file mode 100755
index 0000000..cab942e
--- /dev/null
+++ b/gmcapsuled
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# GmCapsule: Extensible Server for Gemini and Titan
+#
+# Copyright (c) 2022 Jaakko Keränen <jaakko.keranen@iki.fi>
+# License: BSD 2-Clause
+
+import argparse
+import sys
+import threading
+
+from gmcapsule import *
+
+VERSION = '0.1'
+
+print(f"GmCapsule {VERSION}")
+
+# TODO: Parse command arguments.
+
+cfg = Config()
+capsule = Capsule(cfg)
+capsule.run()
diff --git a/modules/400_cgi.py b/modules/400_cgi.py
new file mode 100644
index 0000000..c635a85
--- /dev/null
+++ b/modules/400_cgi.py
@@ -0,0 +1,68 @@
+
+import os
+import subprocess
+import urllib.parse
+
+from gmcapsule import Capsule
+
+
+class CgiContext:
+    def __init__(self, url_path, args):
+        self.args = args
+        self.base_path = url_path
+        if self.base_path.endswith('/*'):
+            self.base_path = self.base_path[:-2]
+
+    def __call__(self, req):
+        try:
+            cfg = Capsule.config()
+            query = urllib.parse.unquote(req.query)
+            env_vars = dict(os.environ)
+
+            # Standard CGI variables.
+            env_vars['REMOTE_ADDR'] = '%s:%d' % req.remote_address
+            env_vars['QUERY_STRING'] = req.query
+            assert req.path.startswith(self.base_path)
+            env_vars['PATH_INFO'] = req.path[len(self.base_path):]
+
+            if req.identity:
+                env_vars['REMOTE_IDENT'] = str(req.identity)
+                env_vars['REMOTE_USER'] = req.identity.subject()
+
+            if req.content:
+                env_vars['TITAN_TOKEN'] = req.content_token if req.content_token is not None else ''
+                env_vars['TITAN_MIME'] = req.content_mime if req.content_mime is not None else ''
+
+            print(req.content)
+
+            result = subprocess.run(self.args, check=True,
+                input=req.content,
+                stdout=subprocess.PIPE,
+                env=env_vars).stdout
+            try:
+                # Parse response header.
+                crlf_pos = result.find(b'\r\n')
+                if crlf_pos >= 1024:
+                    return 42, "CGI command returned invalid response header"
+                header = result[:crlf_pos].decode('utf-8')
+                body = result[crlf_pos + 2:]
+                status = int(header[:2])
+                meta = header[2:].strip()
+                if status < 10:
+                    return 42, "CGI command returned invalid status code"
+                return status, meta, body
+            except:
+                try:
+                    return 20, 'text/plain; charset=utf-8', result.decode('utf-8')
+                except:
+                    return 20, 'application/octet-stream', result
+        except Exception as er:
+            return 42, "CGI error: " + str(er)
+
+
+def init(capsule):
+    cfg = Capsule.config()
+    for protocol in cfg.cgi:
+        for hostname in cfg.cgi[protocol]:
+            for entry in cfg.cgi[protocol][hostname]:
+                capsule.add(entry[0], CgiContext(entry[0], entry[1]), hostname, protocol)
diff --git a/modules/500_static.py b/modules/500_static.py
new file mode 100644
index 0000000..a525993
--- /dev/null
+++ b/modules/500_static.py
@@ -0,0 +1,66 @@
+import fnmatch
+import os.path
+import string
+
+from gmcapsule import Capsule, get_mime_type
+from pathlib import Path
+
+META = '.meta'
+
+
+def check_meta_rules(path, hostname):
+    cfg = Capsule.config()
+    root = (cfg.root_dir / hostname).resolve()
+    dir = path.parent
+    while True:
+        if not str(dir.resolve()).startswith(str(root)):
+            break
+        if (dir / META).exists():
+            for rule in open(dir / META, 'rt').readlines():
+                rule = rule.strip()
+                if len(rule) == 0: continue
+                pos = rule.find(':')
+                if pos < 0: continue
+                rule_path = dir / rule[:pos].strip()
+                rule_meta = rule[pos + 1:].strip()
+                if fnmatch.fnmatch(root / path, rule_path):
+                    if len(rule_meta) >= 4 and rule_meta[2] in string.whitespace:
+                        return int(rule_meta[:2]), rule_meta[3:]
+                    return 20, rule_meta
+        dir = dir.parent
+
+    return 20, get_mime_type(path)
+
+
+def serve_file(req):
+    if req.scheme != 'gemini':
+        return 59, "Only Gemini requests allowed"
+
+    cfg = Capsule.config()
+    if req.path == '':
+        return 30, '/'
+
+    for seg in req.path.split('/'):
+        if seg != '.' and seg != '..' and seg.startswith('.'):
+            return 51, "Not found"
+
+    host_root = (cfg.root_dir / req.hostname).resolve()
+    path = (host_root / req.path[1:]).resolve()
+    if not str(path).startswith(str(host_root)):
+        return 51, "Not found"
+
+    if os.path.isdir(str(path)):
+        path = path / 'index.gmi'
+
+    status, meta = check_meta_rules(path, req.hostname)
+    if status and status != 20:
+        return status, meta
+
+    if not os.path.exists(str(path)):
+        return 51, "Not found"
+
+    return status, meta, (open(path, 'rb').read() if status == 20 else None)
+
+
+def init(capsule):
+    capsule.add('/*', serve_file)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..8c388fa
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+pyOpenSSL