mirror/decomp.me
1
0
Fork 0
mirror of https://github.com/decompme/decomp.me.git synced 2026-05-20 23:29:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add libraries back in

Browse Source
This commit is contained in:
Mark Street
2024-04-14 10:16:42 +01:00
parent 6028b4a398
commit a6085c2818
23 changed files with 185 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/coreapp/compilers.py
-4
View File
@@ -2,7 +2,6 @@ import copy
import logging
import platform as platform_stdlib
from dataclasses import dataclass
from pathlib import Path
from typing import ClassVar, List
from coreapp import platforms
@@ -19,9 +18,6 @@ from django.conf import settings
logger = logging.getLogger(__name__)
CONFIG_PY = "config.py"
COMPILER_BASE_PATH: Path = settings.COMPILER_BASE_PATH
@dataclass(frozen=True)
class Compiler:
backend/coreapp/libraries.py
+1 -54
View File
@@ -1,14 +1,5 @@
from dataclasses import dataclass
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING, Dict
from django.conf import settings
if TYPE_CHECKING:
LIBRARY_BASE_PATH: Path
else:
LIBRARY_BASE_PATH: Path = settings.LIBRARY_BASE_PATH
from typing import Dict
@dataclass(frozen=True)
@@ -16,15 +7,6 @@ class Library:
name: str
version: str
def get_include_path(self, platform: str) -> Path:
return LIBRARY_BASE_PATH / platform / self.name / self.version
def available(self, platform: str) -> bool:
include_path = self.get_include_path(platform)
if not include_path.exists():
print(f"Library {self.name} {self.version} not found at {include_path}")
return include_path.exists()
def to_json(self) -> Dict[str, str]:
library = {
"name": self.name,
@@ -38,38 +20,3 @@ class LibraryVersions:
name: str
supported_versions: list[str]
platform: str
@property
def path(self) -> Path:
return LIBRARY_BASE_PATH / self.platform / self.name
@cache
def available_libraries() -> list[LibraryVersions]:
results = []
for platform_dir in LIBRARY_BASE_PATH.iterdir():
if not platform_dir.is_dir():
continue
for lib_dir in platform_dir.iterdir():
versions = []
if not lib_dir.is_dir():
continue
for version_dir in lib_dir.iterdir():
if not version_dir.is_dir():
continue
if not (version_dir / "include").exists():
continue
versions.append(version_dir.name)
if len(versions) > 0:
results.append(
LibraryVersions(
name=lib_dir.name,
supported_versions=versions,
platform=platform_dir.name,
)
)
return results
backend/coreapp/registry.py
+68 -37
View File
@@ -7,6 +7,8 @@ from django.utils.timezone import now
from rest_framework.exceptions import APIException
from .libraries import LibraryVersions
# FIXME: circular import!
# from .platforms import Platform
# from .compilers import Compiler
@@ -17,10 +19,11 @@ logger = logging.getLogger(__name__)
class ManagedSession:
# TODO: version? os? use_ssl?
def __init__(self, hostname, port, compilers_hash) -> None:
def __init__(self, hostname, port, compilers_hash, libraries_hash) -> None:
self.hostname = hostname
self.port = port
self.compilers_hash = compilers_hash
self.libraries_hash = libraries_hash
self.session = requests.Session()
def __str__(self) -> str:
@@ -52,74 +55,102 @@ class Registry:
PLATFORMS: Dict[str, List[Any]] = dict()
COMPILERS: Dict[str, List[Any]] = dict()
LIBRARIES: Dict[str, List[Any]] = dict()
last_updated = now()
def is_known_host(self, hostname, port):
def is_known_host(self, hostname: str, port: int) -> bool:
return (hostname, port) in self.sessions
def add_host(self, hostname, port, compilers, compilers_hash):
def add_host(
self,
hostname: str,
port: int,
compilers: List[Any],
compilers_hash: str,
libraries: List[Any],
libraries_hash: str,
) -> None:
# FIXME: move to top of the file...
from .compilers import Compiler
session = self.sessions.get((hostname, port))
if session is None or compilers_hash != session.compilers_hash:
if (
session is None
or compilers_hash != session.compilers_hash
or libraries_hash != session.libraries_hash
):
if session is None:
session = ManagedSession(hostname, port, compilers_hash)
session = ManagedSession(hostname, port, compilers_hash, libraries_hash)
update_compilers = update_libraries = True
else:
update_compilers = compilers_hash != session.compilers_hash
update_libraries = libraries_hash != session.libraries_hash
platforms = set()
for compiler_dict in compilers:
try:
compiler = Compiler.from_dict(compiler_dict)
except Exception as e:
logger.error(
"Failed to create Compiler from %s, %s", compiler_dict, e
if update_compilers:
platforms = set()
for compiler_dict in compilers:
try:
compiler = Compiler.from_dict(compiler_dict)
except Exception as e:
logger.error(
"Failed to create Compiler from %s, %s", compiler_dict, e
)
return
if compiler.id not in self.COMPILERS:
self.COMPILERS[compiler.id] = compiler
platforms.add(compiler.platform.id)
# external lookup
if compiler.platform.id not in self.PLATFORMS:
self.PLATFORMS[compiler.platform.id] = compiler.platform
# internal lookup
if compiler.id not in self.compilers:
self.compilers[compiler.id] = []
self.compilers[compiler.id].append(
(hostname, port),
)
return
if compiler.id not in self.COMPILERS:
self.COMPILERS[compiler.id] = compiler
for platform in platforms:
if platform not in self.platforms:
self.platforms[platform] = []
self.platforms[platform].append(
(hostname, port),
)
if update_libraries:
for library in libraries:
library_version = LibraryVersions(**library)
platforms.add(compiler.platform.id)
# external lookup
if compiler.platform.id not in self.PLATFORMS:
self.PLATFORMS[compiler.platform.id] = compiler.platform
# internal lookup
if compiler.id not in self.compilers:
self.compilers[compiler.id] = []
self.compilers[compiler.id].append(
(hostname, port),
)
for platform in platforms:
if platform not in self.platforms:
self.platforms[platform] = []
self.platforms[platform].append(
(hostname, port),
)
if library_version.name not in self.LIBRARIES:
self.LIBRARIES[library_version.name] = library_version
self.sessions[(hostname, port)] = session
self.last_updated = now()
logger.info(
"Successfully registered %s:%i with %i platform(s) and %i compiler(s)",
"Successfully registered %s:%i with %i platform(s), %i compiler(s) and %i library(s)",
hostname,
port,
len(platforms),
len(compilers),
len(libraries),
)
else:
# logger.debug(f"Ignoring '/register' from {hostname}:{port} as compiler_hash is the same ({compilers_hash})")
pass
def available_compilers(self):
def available_compilers(self) -> List[Any]:
return list(sorted(self.COMPILERS.values(), key=lambda x: x.id))
def available_platforms(self):
def available_platforms(self) -> List[Any]:
return list(sorted(self.PLATFORMS.values(), key=lambda x: x.name))
def available_libraries(self) -> List[LibraryVersions]:
return list(sorted(self.LIBRARIES.values(), key=lambda x: x.name))
def get_compiler_by_id(self, compiler_id):
compiler = self.COMPILERS.get(compiler_id)
if not compiler:
backend/coreapp/sandbox.py
+1 -115
View File
@@ -1,19 +1,11 @@
import contextlib
import logging
# import os
# import shlex
# import subprocess
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional
from django.conf import settings
# from coreapp.error import SandboxError
logger = logging.getLogger(__name__)
class Sandbox(contextlib.AbstractContextManager["Sandbox"]):
def __enter__(self) -> "Sandbox":
@@ -32,109 +24,3 @@ class Sandbox(contextlib.AbstractContextManager["Sandbox"]):
def __exit__(self, *exc: Any) -> None:
self.temp_dir.cleanup()
# The meat of Sandbox has moved to platforms/src/sandbox
# @staticmethod
# def quote_options(opts: str) -> str:
# return shlex.join(shlex.split(opts))
# def rewrite_path(self, path: Path) -> str:
# if self.use_jail and path.is_relative_to(self.path):
# path = Path("/tmp") / path.relative_to(self.path)
# return str(path)
# def sandbox_command(self, mounts: List[Path], env: Dict[str, str]) -> List[str]:
# if not self.use_jail:
# return []
# settings.SANDBOX_CHROOT_PATH.mkdir(parents=True, exist_ok=True)
# settings.WINEPREFIX.mkdir(parents=True, exist_ok=True)
# assert ":" not in str(self.path)
# assert ":" not in str(settings.WINEPREFIX)
# # fmt: off
# wrapper = [
# str(settings.SANDBOX_NSJAIL_BIN_PATH),
# "--mode", "o",
# "--chroot", str(settings.SANDBOX_CHROOT_PATH),
# "--bindmount", f"{self.path}:/tmp",
# "--bindmount", f"{self.path}:/run/user/{os.getuid()}",
# "--bindmount_ro", "/dev",
# "--bindmount_ro", "/bin",
# "--bindmount_ro", "/etc/alternatives",
# "--bindmount_ro", "/etc/fonts",
# "--bindmount_ro", "/etc/passwd",
# "--bindmount_ro", "/lib",
# "--bindmount_ro", "/lib64",
# "--bindmount_ro", "/usr",
# "--bindmount_ro", "/proc",
# "--bindmount", f"{self.path}:/var/tmp",
# "--bindmount_ro", str(settings.COMPILER_BASE_PATH),
# "--bindmount_ro", str(settings.LIBRARY_BASE_PATH),
# "--env", "PATH=/usr/bin:/bin",
# "--cwd", "/tmp",
# "--rlimit_fsize", "soft",
# "--rlimit_nofile", "soft",
# # the following are settings that can be removed once we are done with wine
# "--bindmount_ro", f"{settings.WINEPREFIX}:/wine",
# "--env", "WINEDEBUG=-all",
# "--env", "WINEPREFIX=/wine",
# ]
# # fmt: on
# if settings.SANDBOX_DISABLE_PROC:
# wrapper.append("--disable_proc") # needed for running inside Docker
# if not settings.DEBUG:
# wrapper.append("--really_quiet")
# for mount in mounts:
# wrapper.extend(["--bindmount_ro", str(mount)])
# for key in env:
# wrapper.extend(["--env", key])
# wrapper.append("--")
# return wrapper
# def run_subprocess(
# self,
# args: Union[str, List[str]],
# *,
# mounts: Optional[List[Path]] = None,
# env: Optional[Dict[str, str]] = None,
# shell: bool = False,
# timeout: Optional[float] = None,
# ) -> subprocess.CompletedProcess[str]:
# mounts = mounts if mounts is not None else []
# env = env if env is not None else {}
# timeout = None if timeout == 0 else timeout
# try:
# wrapper = self.sandbox_command(mounts, env)
# except Exception as e:
# raise SandboxError(f"Failed to initialize sandbox command: {e}")
# if shell:
# if isinstance(args, list):
# args = " ".join(args)
# command = wrapper + ["/bin/bash", "-euo", "pipefail", "-c", args]
# else:
# assert isinstance(args, list)
# command = wrapper + args
# debug_env_str = " ".join(
# f"{key}={shlex.quote(value)}" for key, value in env.items() if key != "PATH"
# )
# logger.debug(f"Sandbox Command: {debug_env_str} {shlex.join(command)}")
# return subprocess.run(
# command,
# text=True,
# errors="backslashreplace",
# env=env,
# cwd=self.path,
# check=True,
# shell=False,
# stdout=subprocess.PIPE,
# stderr=subprocess.STDOUT,
# timeout=timeout,
# )
backend/coreapp/views/library.py
+4 -9
View File
@@ -1,16 +1,11 @@
from typing import Dict
from django.utils.timezone import now
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.views import APIView
from coreapp import libraries
from coreapp.registry import registry
from ..decorators.django import condition
boot_time = now()
class LibraryDetail(APIView):
@staticmethod
@@ -21,15 +16,15 @@ class LibraryDetail(APIView):
"supported_versions": l.supported_versions,
"platform": l.platform,
}
for l in libraries.available_libraries()
for l in registry.available_libraries()
if platform == "" or l.platform == platform
]
@condition(last_modified_func=lambda request: boot_time)
@condition(last_modified_func=lambda request: registry.last_updated)
def head(self, request: Request) -> Response:
return Response()
@condition(last_modified_func=lambda request: boot_time)
@condition(last_modified_func=lambda request: registry.last_updated)
def get(self, request: Request) -> Response:
platform = request.query_params.get("platform", "")
return Response(
backend/coreapp/views/register.py
+6 -1
View File
@@ -45,8 +45,13 @@ class Register(APIView):
compilers = payload.get("compilers")
compilers_hash = payload.get("compilers_hash")
libraries = payload.get("libraries")
libraries_hash = payload.get("libraries_hash")
if compilers is not None and compilers_hash is not None:
registry.add_host(hostname, port, compilers, compilers_hash)
registry.add_host(
hostname, port, compilers, compilers_hash, libraries, libraries_hash
)
return Response("Ping/Pong!", status=status.HTTP_200_OK)
if registry.is_known_host(hostname, int(port)):
backend/decompme/settings.py
-8
View File
@@ -32,10 +32,7 @@ env = environ.Env(
SESSION_COOKIE_SECURE=(bool, True),
GITHUB_CLIENT_ID=(str, ""),
GITHUB_CLIENT_SECRET=(str, ""),
COMPILER_BASE_PATH=(str, BASE_DIR / "compilers"),
LIBRARY_BASE_PATH=(str, BASE_DIR / "libraries"),
COMPILATION_CACHE_SIZE=(int, 100),
WINEPREFIX=(str, "/tmp/wine"),
COMPILATION_TIMEOUT_SECONDS=(int, 10),
ASSEMBLY_TIMEOUT_SECONDS=(int, 3),
OBJDUMP_TIMEOUT_SECONDS=(int, 3),
@@ -201,9 +198,6 @@ if DEBUG:
else:
SESSION_COOKIE_SAMESITE = "Lax"
COMPILER_BASE_PATH = Path(env("COMPILER_BASE_PATH"))
LIBRARY_BASE_PATH = Path(env("LIBRARY_BASE_PATH"))
USE_SANDBOX_JAIL = env("USE_SANDBOX_JAIL")
SANDBOX_NSJAIL_BIN_PATH = Path(env("SANDBOX_NSJAIL_BIN_PATH"))
SANDBOX_CHROOT_PATH = BASE_DIR.parent / "sandbox" / "root"
@@ -215,8 +209,6 @@ GITHUB_CLIENT_SECRET = env("GITHUB_CLIENT_SECRET", str)
COMPILATION_CACHE_SIZE = env("COMPILATION_CACHE_SIZE", int)
WINEPREFIX = Path(env("WINEPREFIX"))
TIMEOUT_SCALE_FACTOR = env("TIMEOUT_SCALE_FACTOR", int)
COMPILATION_TIMEOUT_SECONDS = (
env("COMPILATION_TIMEOUT_SECONDS", int) * TIMEOUT_SCALE_FACTOR
docker-platform-template.yaml
-1
View File
@@ -21,7 +21,6 @@ services:
env_file:
- ./platforms/docker/dev.env
volumes:
- ./backend/libraries:/backend/libraries
- ./platforms:/platforms
tmpfs:
# explicitly mount /dev/shm with exec as dosemu2 requires exec privilege
platforms/TODO.md
+2 -2
View File
@@ -19,9 +19,9 @@ Currently Platforms are defined both in platforms/ and backend/
## Things to do
- Add Libraries support back in
- move /register 'secret' into environment variables
- delete session if platform becomes unavailable
- delete session if platform becomes unavailable?
- remove DUMMY compilers from 'platforms' ? I think they are only needed for testing backend?
## Open questions
- Should Platforms belong to both "backend" and "platforms"?
platforms/docker_entrypoint.sh
+3 -1
View File
@@ -5,6 +5,8 @@
# e.g. if a temp file exists and is newer than e.g. 5 minutes, we assume another process is running and fast exit
# otherwise touch the temp file and continue, and delete the temp file at the end
# TOOD: wrap this all in a if guard so we do not do it every time we start up when developing...
if [[ "${SUPPORTED_PLATFORMS}x" == "x" ]]; then
echo "Downloading all compilers/libraries..."
# python3 compilers/download.py
@@ -17,6 +19,6 @@ else
fi
# do we need to wait for anything to come up?
# should we wait for backend to become available?
python3 main.py
backend/libraries/.gitignore → platforms/libraries/.gitignore
View File
backend/libraries/download.py → platforms/libraries/download.py
View File
backend/libraries/libraries.yaml → platforms/libraries/libraries.yaml
View File
platforms/main.py
+17 -1
View File
@@ -18,6 +18,7 @@ from src.handlers.compile import CompileHandler
from src.handlers.assemble import AssembleHandler
from src.handlers.objdump import ObjdumpHandler
from src.compilers import available_platforms, available_compilers
from src.libraries import available_libraries
logger = logging.getLogger(__file__)
@@ -41,16 +42,30 @@ class CompilersHandler(tornado.web.RequestHandler):
)
class LibrariesHandler(tornado.web.RequestHandler):
def get(self):
self.write(
json.dumps(
[library_version.to_json() for library_version in available_libraries()]
)
)
def register_with_backend():
compilers = [c.to_json() for c in available_compilers()]
compilers_hash = hashlib.sha256(json.dumps(compilers).encode("utf")).hexdigest()
libraries = [l.to_json() for l in available_libraries()]
libraries_hash = hashlib.sha256(json.dumps(libraries).encode("utf")).hexdigest()
data = {
"key": "secret",
"hostname": platform.node(),
"port": settings.PORT,
"compilers": compilers,
"compilers_hash": compilers_hash,
"libraries": libraries,
"libraries_hash": libraries_hash,
}
try:
url = f"{settings.INTERNAL_API_BASE}/register"
@@ -59,7 +74,7 @@ def register_with_backend():
assert res.status_code in (200, 201), "status_code should be 200 or 201"
if res.status_code == 201:
logger.info("Backend did not know about us...")
logger.info("Successfully registered with backend")
except Exception as e:
logger.warning("register_with_backend raised exception: %s", e)
@@ -86,6 +101,7 @@ def main():
(r"/objdump", ObjdumpHandler),
(r"/platforms", PlatformsHandler),
(r"/compilers", CompilersHandler),
(r"/libraries", LibrariesHandler),
]
ioloop = tornado.ioloop.IOLoop.current()
platforms/src/compilers.py
+2 -5
View File
@@ -6,7 +6,7 @@ from dataclasses import dataclass
from functools import cache
from typing import ClassVar, List
from tornado.options import options as settings
from .settings import settings, is_supported_platform
from .models.compiler import Compiler
from .models.platform import Platform
@@ -153,13 +153,10 @@ def from_id(compiler_id: str) -> Compiler:
@cache
def available_compilers() -> List[Compiler]:
if not settings.SUPPORTED_PLATFORMS:
return list(_compilers.values())
supported_platforms = settings.SUPPORTED_PLATFORMS.split(",")
return [
compiler
for compiler in _compilers.values()
if compiler.platform.id in supported_platforms
if is_supported_platform(compiler.platform.id)
]
platforms/src/handlers/assemble.py
+1 -3
View File
@@ -1,6 +1,5 @@
import json
import logging
import os
import subprocess
import base64
@@ -12,8 +11,7 @@ from .compile import PATH
from ..models.requests import AssembleRequest
from ..sandbox import Sandbox
from tornado.options import options as settings
from ..settings import settings
logger = logging.getLogger(__file__)
platforms/src/handlers/compile.py
+4 -1
View File
@@ -7,10 +7,10 @@ import subprocess
import time
import tornado
from tornado.options import options as settings
from ..models.requests import CompileRequest
from ..sandbox import Sandbox
from ..settings import settings
logger = logging.getLogger(__file__)
@@ -115,6 +115,9 @@ class CompileHandler(tornado.web.RequestHandler):
for lib in compile_request.libraries
)
)
logger.info("Hello")
logger.info("libraries_compiler_flags: %s", libraries_compiler_flags)
compile_proc = sandbox.run_subprocess(
cc_cmd,
mounts=([compiler.path] if compiler.platform.id != "dummy" else []),
platforms/src/handlers/objdump.py
+1 -2
View File
@@ -8,7 +8,6 @@ from typing import List
from pathlib import Path
import tornado
from tornado.options import options as settings
from .compile import PATH
@@ -16,7 +15,7 @@ from ..models.platform import Platform
from ..models.requests import ObjdumpRequest
from ..sandbox import Sandbox
from ..settings import settings
logger = logging.getLogger(__file__)
platforms/src/libraries.py
+38 -1
View File
@@ -1 +1,38 @@
# TODO
from functools import cache
from .settings import is_supported_platform, settings
from .models.library import LibraryVersions
@cache
def available_libraries() -> list[LibraryVersions]:
results = []
for platform_dir in settings.LIBRARY_BASE_PATH.iterdir():
if not platform_dir.is_dir():
continue
if not is_supported_platform(platform_dir.name):
continue
for lib_dir in platform_dir.iterdir():
versions = []
if not lib_dir.is_dir():
continue
for version_dir in lib_dir.iterdir():
if not version_dir.is_dir():
continue
if not (version_dir / "include").exists():
continue
versions.append(version_dir.name)
if len(versions) > 0:
results.append(
LibraryVersions(
name=lib_dir.name,
supported_versions=versions,
platform=platform_dir.name,
)
)
return results
platforms/src/models/compiler.py
+2 -2
View File
@@ -4,11 +4,11 @@ from dataclasses import dataclass
from typing import Optional, ClassVar
from pathlib import Path
from tornado.options import options as settings
from .platform import Platform
from .flags import Flags, Language
from ..settings import settings
logger = logging.getLogger(__file__)
platforms/src/models/library.py
+19 -1
View File
@@ -1,7 +1,7 @@
from dataclasses import dataclass
from pathlib import Path
from tornado.options import options as settings
from ..settings import settings
@dataclass(frozen=True)
@@ -17,3 +17,21 @@ class Library:
if not include_path.exists():
print(f"Library {self.name} {self.version} not found at {include_path}")
return include_path.exists()
@dataclass(frozen=True)
class LibraryVersions:
name: str
supported_versions: list[str]
platform: str
@property
def path(self) -> Path:
return settings.LIBRARY_BASE_PATH / self.platform / self.name
def to_json(self):
return {
"name": self.name,
"supported_versions": self.supported_versions,
"platform": self.platform,
}
platforms/src/sandbox.py
+1 -2
View File
@@ -7,8 +7,7 @@ from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Dict, List, Optional, Union
from tornado.options import options as settings
from .settings import settings
logger = logging.getLogger(__name__)
platforms/src/settings.py
+15 -5
View File
@@ -1,7 +1,10 @@
import os
from pathlib import Path
import tornado.options
from tornado.options import (
options as settings,
define as tornado_define,
)
def truthy(x):
@@ -16,11 +19,18 @@ def define(name, default=None, type=str):
else:
setting = truthy(setting) if type is bool else type(setting)
tornado.options.define(name, setting, type)
# return value to help unittests
tornado_define(name, setting, type)
return (name, setting, type)
def is_supported_platform(platform_id):
if settings.SUPPORTED_PLATFORMS is None:
return True
return platform_id in settings.SUPPORTED_PLATFORMS.split(",")
BASE_DIR = Path(__file__).resolve().parent.parent
define("PORT", default=9000, type=int)
define("MAX_WORKERS", default=32, type=int)
@@ -40,8 +50,8 @@ define("SANDBOX_DISABLE_PROC", default=True, type=bool)
define("WINEPREFIX", default=Path("/tmp/wine"), type=Path)
define("COMPILER_BASE_PATH", default=Path("/platforms/compilers"), type=Path)
define("LIBRARY_BASE_PATH", default=Path("/backend/libraries"), type=Path)
define("COMPILER_BASE_PATH", default=BASE_DIR / "compilers", type=Path)
define("LIBRARY_BASE_PATH", default=BASE_DIR / "libraries", type=Path)
define("COMPILATION_TIMEOUT_SECONDS", default=30, type=int)
define("ASSEMBLY_TIMEOUT_SECONDS", default=30, type=int)