mirror/decomp.me
1
0
Fork 0
mirror of https://github.com/decompme/decomp.me.git synced 2026-05-24 09:39:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Platforms now belong to 'platforms'

Browse Source
This commit is contained in:
Mark Street
2024-04-14 13:09:48 +01:00
parent a6085c2818
commit aebedc71ad
21 changed files with 186 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/coreapp/compiler_wrapper.py
+4 -4
View File
@@ -15,8 +15,8 @@ from typing import (
from django.conf import settings
from coreapp import platforms
from coreapp.platforms import Platform
from coreapp.platforms import Platform, DUMMY_PLATFORM
from coreapp.compilers import DUMMY_COMPILER
from coreapp.registry import registry
@@ -102,7 +102,7 @@ class CompilerWrapper:
function: str = "",
libraries: Sequence[Library] = (),
) -> CompilationResult:
if compiler_id == "DUMMY":
if compiler_id == DUMMY_COMPILER.id:
return CompilationResult(f"compiled({context}\n{code}".encode("UTF-8"), "")
session = registry.get_session_for_compiler(compiler_id)
@@ -154,7 +154,7 @@ class CompilerWrapper:
logger.debug(f"Assembly cache hit! hash: {hash}")
return cached_assembly
if platform == platforms.DUMMY:
if platform == DUMMY_PLATFORM:
assembly = Assembly(
hash=hash,
arch=platform.arch,
backend/coreapp/compilers.py
+12 -13
View File
@@ -1,22 +1,16 @@
import copy
import logging
import platform as platform_stdlib
from dataclasses import dataclass
from typing import ClassVar, List
from coreapp import platforms
from django.conf import settings
from coreapp.platforms import Platform, DUMMY_PLATFORM
from coreapp.flags import (
Flags,
Flag,
Language,
)
from coreapp.platforms import (
Platform,
from_id as platform_from_id,
)
from django.conf import settings
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@@ -31,9 +25,14 @@ class Compiler:
@staticmethod
def from_dict(compiler_dict):
# FIXME: circular dependency
from .registry import registry
compiler_dict = copy.deepcopy(compiler_dict)
compiler_dict["platform"] = platform_from_id(compiler_dict["platform"])
compiler_dict["platform"] = registry.get_platform_by_id(
compiler_dict["platform"]
)
compiler_dict["flags"] = [
Flag.from_dict(flag) for flag in compiler_dict["flags"]
]
@@ -57,8 +56,8 @@ class DummyLongRunningCompiler(DummyCompiler):
return settings.DUMMY_COMPILER and platform_stdlib.system() != "Windows"
DUMMY = DummyCompiler(id="dummy", platform=platforms.DUMMY)
DUMMY_COMPILER = DummyCompiler(id="dummy", platform=DUMMY_PLATFORM)
DUMMY_LONGRUNNING = DummyLongRunningCompiler(
id="dummy_longrunning", platform=platforms.DUMMY
DUMMY_COMPILER_LONGRUNNING = DummyLongRunningCompiler(
id="dummy_longrunning", platform=DUMMY_PLATFORM
)
backend/coreapp/decompiler_wrapper.py
+2 -3
View File
@@ -1,7 +1,6 @@
import logging
from coreapp import compilers
from coreapp.compilers import Compiler
from coreapp.compilers import Compiler, DUMMY_COMPILER
from coreapp.m2c_wrapper import M2CError, M2CWrapper
from coreapp.platforms import Platform
@@ -22,7 +21,7 @@ class DecompilerWrapper:
context: str,
compiler: Compiler,
) -> str:
if compiler == compilers.DUMMY:
if compiler == DUMMY_COMPILER:
return f"decompiled({asm})"
ret = default_source_code
backend/coreapp/diff_wrapper.py
+3 -3
View File
@@ -6,7 +6,7 @@ from functools import lru_cache
import diff as asm_differ
from coreapp.platforms import DUMMY, Platform
from coreapp.platforms import Platform, DUMMY_PLATFORM
from coreapp.flags import ASMDIFF_FLAG_PREFIX
from coreapp.registry import registry
@@ -204,8 +204,8 @@ class DiffWrapper:
compiled_elf: bytes,
diff_flags: List[str],
) -> DiffResult:
if platform == DUMMY:
# Todo produce diff for dummy
if platform == DUMMY_PLATFORM:
# TODO: produce diff for dummy
return DiffResult({"rows": ["a", "b"]}, "")
try:
backend/coreapp/platforms.py
+18 -151
View File
@@ -1,15 +1,10 @@
import logging
import copy
from dataclasses import dataclass, field
from typing import Any, Dict, OrderedDict
from typing import Any, Dict
from coreapp.flags import COMMON_DIFF_FLAGS, COMMON_MIPS_DIFF_FLAGS, Flags
from coreapp.models.preset import Preset
from coreapp.models.scratch import Scratch
from rest_framework.exceptions import APIException
from coreapp.serializers import PresetSerializer
logger = logging.getLogger(__name__)
from coreapp.flags import COMMON_DIFF_FLAGS, Flags, Flag
@dataclass(frozen=True)
@@ -17,7 +12,7 @@ class Platform:
id: str
name: str
description: str
arch: str # used by asm-differ
arch: str
diff_flags: Flags = field(default_factory=lambda: COMMON_DIFF_FLAGS, hash=False)
has_decompiler: bool = False
@@ -35,6 +30,9 @@ class Platform:
"has_decompiler": self.has_decompiler,
}
if include_presets:
# FIXME: circular dependency if imported at top level
from coreapp.serializers import PresetSerializer
ret["presets"] = [
PresetSerializer(p).data
for p in Preset.objects.filter(platform=self.id).order_by("name")
@@ -43,154 +41,23 @@ class Platform:
ret["num_scratches"] = self.get_num_scratches()
return ret
@staticmethod
def from_dict(platform_dict):
platform_dict = copy.deepcopy(platform_dict)
def from_id(platform_id: str) -> Platform:
# from coreapp.registry import registry
if "diff_flags" not in platform_dict or len(platform_dict["diff_flags"]) == 0:
platform_dict["diff_flags"] = COMMON_DIFF_FLAGS
else:
platform_dict["diff_flags"] = [
Flag.from_dict(flag) for flag in platform_dict["diff_flags"]
]
# available = registry.available_platforms()
# if platform_id not in available:
# raise APIException(f"Unknown platform: {platform_id}")
# return available[platform_id]
platform = _platforms.get(platform_id)
if platform is None:
raise APIException(f"Unknown platform: {platform_id}")
return platform
return Platform(**platform_dict)
DUMMY = Platform(
DUMMY_PLATFORM = Platform(
id="dummy",
name="Dummy System",
description="DMY",
arch="dummy",
)
MSDOS = Platform(
id="msdos",
name="Microsoft DOS",
description="x86",
arch="i686",
)
WIN32 = Platform(
id="win32",
name="Windows (9x/NT)",
description="x86 (32bit)",
arch="i686",
)
SWITCH = Platform(
id="switch",
name="Nintendo Switch",
description="ARMv8-A",
arch="aarch64",
)
N64 = Platform(
id="n64",
name="Nintendo 64",
description="MIPS (big-endian)",
arch="mips",
diff_flags=COMMON_DIFF_FLAGS + COMMON_MIPS_DIFF_FLAGS,
has_decompiler=True,
)
IRIX = Platform(
id="irix",
name="IRIX",
description="MIPS (big-endian, PIC)",
arch="mips",
diff_flags=COMMON_DIFF_FLAGS + COMMON_MIPS_DIFF_FLAGS,
has_decompiler=True,
)
PS1 = Platform(
id="ps1",
name="PlayStation",
description="MIPS (little-endian)",
arch="mipsel",
diff_flags=COMMON_DIFF_FLAGS + COMMON_MIPS_DIFF_FLAGS,
has_decompiler=True,
)
PSP = Platform(
id="psp",
name="PlayStation Portable",
description="MIPS (little-endian)",
arch="mipsel:4000",
diff_flags=COMMON_DIFF_FLAGS + COMMON_MIPS_DIFF_FLAGS,
has_decompiler=True,
)
SATURN = Platform(
id="saturn",
name="Saturn",
description="SH2 (big-endian)",
arch="sh2",
diff_flags=COMMON_DIFF_FLAGS,
)
PS2 = Platform(
id="ps2",
name="PlayStation 2",
description="MIPS (little-endian)",
arch="mipsee",
diff_flags=COMMON_DIFF_FLAGS + COMMON_MIPS_DIFF_FLAGS,
has_decompiler=True,
)
MACOSX = Platform(
id="macosx",
name="Mac OS X",
description="PowerPC",
arch="ppc",
)
GC_WII = Platform(
id="gc_wii",
name="GameCube / Wii",
description="PowerPC",
arch="ppc",
has_decompiler=True,
)
NDS_ARM9 = Platform(
id="nds_arm9",
name="Nintendo DS",
description="ARMv5TE",
arch="arm32",
)
GBA = Platform(
id="gba",
name="Game Boy Advance",
description="ARMv4T",
arch="arm32",
)
N3DS = Platform(
id="n3ds",
name="Nintendo 3DS",
description="ARMv6K",
arch="arm32",
)
# TODO: _platforms should be populated dynamically based on what platforms are available
_platforms: OrderedDict[str, Platform] = OrderedDict(
{
"dummy": DUMMY,
"irix": IRIX,
"n64": N64,
"gc_wii": GC_WII,
"switch": SWITCH,
"gba": GBA,
"nds_arm9": NDS_ARM9,
"n3ds": N3DS,
"ps1": PS1,
"ps2": PS2,
"psp": PSP,
"saturn": SATURN,
"macosx": MACOSX,
"msdos": MSDOS,
"win32": WIN32,
}
)
backend/coreapp/registry.py
+58 -36
View File
@@ -9,9 +9,8 @@ from rest_framework.exceptions import APIException
from .libraries import LibraryVersions
# FIXME: circular import!
# from .platforms import Platform
# from .compilers import Compiler
from .platforms import Platform, DUMMY_PLATFORM
from .compilers import Compiler, DUMMY_COMPILER
logger = logging.getLogger(__name__)
@@ -19,9 +18,12 @@ logger = logging.getLogger(__name__)
class ManagedSession:
# TODO: version? os? use_ssl?
def __init__(self, hostname, port, compilers_hash, libraries_hash) -> None:
def __init__(
self, hostname, port, platforms_hash, compilers_hash, libraries_hash
) -> None:
self.hostname = hostname
self.port = port
self.platforms_hash = platforms_hash
self.compilers_hash = compilers_hash
self.libraries_hash = libraries_hash
self.session = requests.Session()
@@ -53,42 +55,64 @@ class Registry:
# compiler_id -> [(hostname, port), ...]
compilers: Dict[str, List[tuple[str, int]]] = dict()
PLATFORMS: Dict[str, List[Any]] = dict()
COMPILERS: Dict[str, List[Any]] = dict()
LIBRARIES: Dict[str, List[Any]] = dict()
PLATFORMS: Dict[str, List[Platform]] = dict()
COMPILERS: Dict[str, List[Compiler]] = dict()
LIBRARIES: Dict[str, List[LibraryVersions]] = dict()
last_updated = now()
def is_known_host(self, hostname: str, port: int) -> bool:
return (hostname, port) in self.sessions
def add_host(
def register_host(
self,
hostname: str,
port: int,
platforms: List[Any],
platforms_hash: str,
compilers: List[Any],
compilers_hash: str,
libraries: List[Any],
libraries_hash: str,
) -> None:
# FIXME: move to top of the file...
from .compilers import Compiler
session = self.sessions.get((hostname, port))
if (
session is None
or platforms_hash != session.platforms_hash
or compilers_hash != session.compilers_hash
or libraries_hash != session.libraries_hash
):
if session is None:
session = ManagedSession(hostname, port, compilers_hash, libraries_hash)
update_compilers = update_libraries = True
session = ManagedSession(
hostname, port, platforms_hash, compilers_hash, libraries_hash
)
update_platforms = update_compilers = update_libraries = True
else:
update_platforms = platforms_hash != session.platforms_hash
update_compilers = compilers_hash != session.compilers_hash
update_libraries = libraries_hash != session.libraries_hash
if update_platforms:
for platform_dict in platforms:
try:
platform = Platform.from_dict(platform_dict)
except Exception as e:
logger.error(
"Failed to create Platform from %s, %s", platform_dict, e
)
return False
# assume new/updated data, so overwrite existing
self.PLATFORMS[platform.id] = platform
if platform.id not in self.platforms:
self.platforms[platform.id] = []
self.platforms[platform.id].append(
(hostname, port),
)
if update_compilers:
platforms = set()
for compiler_dict in compilers:
try:
compiler = Compiler.from_dict(compiler_dict)
@@ -96,16 +120,10 @@ class Registry:
logger.error(
"Failed to create Compiler from %s, %s", compiler_dict, e
)
return
return False
if compiler.id not in self.COMPILERS:
self.COMPILERS[compiler.id] = compiler
platforms.add(compiler.platform.id)
# external lookup
if compiler.platform.id not in self.PLATFORMS:
self.PLATFORMS[compiler.platform.id] = compiler.platform
# assume new/updated data, so overwrite existing
self.COMPILERS[compiler.id] = compiler
# internal lookup
if compiler.id not in self.compilers:
@@ -114,15 +132,13 @@ class Registry:
(hostname, port),
)
for platform in platforms:
if platform not in self.platforms:
self.platforms[platform] = []
self.platforms[platform].append(
(hostname, port),
)
if update_libraries:
for library in libraries:
library_version = LibraryVersions(**library)
try:
library_version = LibraryVersions(**library)
except Exception as e:
logger.error("Failed to create Library from %s, %s", library, e)
return False
if library_version.name not in self.LIBRARIES:
self.LIBRARIES[library_version.name] = library_version
@@ -131,16 +147,14 @@ class Registry:
self.last_updated = now()
logger.info(
"Successfully registered %s:%i with %i platform(s), %i compiler(s) and %i library(s)",
"Successfully registered host %s:%i with %i platform(s), %i compiler(s) and %i library(s)",
hostname,
port,
len(platforms),
len(compilers),
len(libraries),
)
else:
# logger.debug(f"Ignoring '/register' from {hostname}:{port} as compiler_hash is the same ({compilers_hash})")
pass
return True
def available_compilers(self) -> List[Any]:
return list(sorted(self.COMPILERS.values(), key=lambda x: x.id))
@@ -152,15 +166,23 @@ class Registry:
return list(sorted(self.LIBRARIES.values(), key=lambda x: x.name))
def get_compiler_by_id(self, compiler_id):
if compiler_id == "dummy":
return DUMMY_COMPILER
compiler = self.COMPILERS.get(compiler_id)
if not compiler:
raise APIException(f"{compiler_id} not known/available")
logger.warning("No known compiler with id %s", compiler_id)
return None
return compiler
def get_platform_by_id(self, platform_id):
if platform_id == "dummy":
return DUMMY_PLATFORM
platform = self.PLATFORMS.get(platform_id)
if not platform:
raise APIException(f"{platform_id} not known/available")
logger.warning("No known platform with id %s", platform_id)
# raise APIException(f"{platform_id} not known/available")
return platform
def get_session_for_platform(self, platform_id) -> ManagedSession:
backend/coreapp/serializers.py
+11 -8
View File
@@ -9,9 +9,6 @@ from rest_framework.fields import SerializerMethodField
from rest_framework.relations import HyperlinkedRelatedField, SlugRelatedField
from rest_framework.reverse import reverse
from coreapp import platforms
from .registry import registry
from .flags import LanguageFlagSet
from .libraries import Library
from .middleware import Request
@@ -21,6 +18,8 @@ from .models.profile import Profile
from .models.project import Project, ProjectMember
from .models.scratch import Scratch
from .registry import registry
def serialize_profile(request: Request, profile: Profile) -> Dict[str, Any]:
if profile.user is None:
@@ -91,7 +90,7 @@ class PresetSerializer(serializers.ModelSerializer[Preset]):
def validate_platform(self, platform: str) -> str:
try:
platforms.from_id(platform)
registry.get_platform_by_id(platform)
except:
raise serializers.ValidationError(f"Unknown platform: {platform}")
return platform
@@ -105,7 +104,7 @@ class PresetSerializer(serializers.ModelSerializer[Preset]):
def validate(self, data: Dict[str, Any]) -> Dict[str, Any]:
compiler = registry.get_compiler_by_id(data["compiler"])
platform = platforms.from_id(data["platform"])
platform = registry.get_platform_by_id(data["platform"])
if compiler.platform != platform:
raise serializers.ValidationError(
@@ -135,7 +134,7 @@ class ScratchCreateSerializer(serializers.Serializer[None]):
def validate_platform(self, platform: str) -> str:
try:
platforms.from_id(platform)
registry.get_platform_by_id(platform)
except:
raise serializers.ValidationError(f"Unknown platform: {platform}")
return platform
@@ -151,7 +150,7 @@ class ScratchCreateSerializer(serializers.Serializer[None]):
if "preset" in data:
preset: Preset = data["preset"]
# Preset dictates platform
data["platform"] = platforms.from_id(preset.platform)
data["platform"] = registry.get_platform_by_id(preset.platform)
if "compiler" not in data or not data["compiler"]:
data["compiler"] = preset.compiler
@@ -181,7 +180,7 @@ class ScratchCreateSerializer(serializers.Serializer[None]):
data["platform"] = compiler.platform
else:
try:
platform = platforms.from_id(data["platform"])
platform = registry.get_platform_by_id(data["platform"])
except APIException:
raise serializers.ValidationError(
f"Unknown platform: {data['platform']}"
@@ -228,6 +227,10 @@ class ScratchSerializer(serializers.ModelSerializer[Scratch]):
- Otherwise, fallback to the compiler's default language
"""
compiler = registry.get_compiler_by_id(scratch.compiler)
if compiler is None:
# warning?
return None
language_flag_set = next(
iter([i for i in compiler.flags if isinstance(i, LanguageFlagSet)]),
None,
backend/coreapp/tests/common.py
+2 -2
View File
@@ -27,8 +27,8 @@ class BaseTestCase(APITestCase):
def create_nop_scratch(self) -> Scratch:
scratch_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "jr $ra\nnop\n",
}
backend/coreapp/tests/test_compilation.py
+1 -1
View File
@@ -221,7 +221,7 @@ nop
"""
result = CompilerWrapper.compile_code(
compilers.DUMMY, "", "sample text 123", ""
compilers.DUMMY_COMPILER, "", "sample text 123", ""
)
self.assertGreater(
len(result.elf_object), 0, "The compilation result should be non-null"
backend/coreapp/tests/test_request.py
+5 -5
View File
@@ -30,13 +30,13 @@ class RequestTests(APITestCase):
class TimeoutTests(BaseTestCase):
@requiresCompiler(compilers.DUMMY_LONGRUNNING)
@requiresCompiler(compilers.DUMMY_COMPILER_LONGRUNNING)
def test_compiler_timeout(self) -> None:
# Test that a hanging compilation will fail with a timeout error
with self.settings(COMPILATION_TIMEOUT_SECONDS=3):
scratch_dict = {
"compiler": compilers.DUMMY_LONGRUNNING.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER_LONGRUNNING.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "asm(AAAAAAAA)",
}
@@ -45,7 +45,7 @@ class TimeoutTests(BaseTestCase):
compile_dict = {
"slug": scratch.slug,
"compiler": compilers.DUMMY_LONGRUNNING.id,
"compiler": compilers.DUMMY_COMPILER_LONGRUNNING.id,
"compiler_flags": "",
"source_code": "source(AAAAAAAA)",
}
@@ -58,7 +58,7 @@ class TimeoutTests(BaseTestCase):
self.assertIn("timeout expired", response.json()["compiler_output"].lower())
# if we don't have DUMMY_LONGRUNNING, it means we'll be unable to use sandbox.run_subprocess
@requiresCompiler(compilers.DUMMY_LONGRUNNING)
@requiresCompiler(compilers.DUMMY_COMPILER_LONGRUNNING)
def test_zero_timeout(self) -> None:
# Tests that passing a timeout of zero to sandbox.run_subprocess will equate
# to disabling the timeout entirely
backend/coreapp/tests/test_scratch.py
+14 -14
View File
@@ -84,8 +84,8 @@ nop
Ensure that we can create scratches with the dummy platform and compiler
"""
scratch_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "this is some test asm",
}
@@ -262,8 +262,8 @@ class ScratchForkTests(BaseTestCase):
Ensure that a scratch's fork maintains the relevant properties of its parent
"""
scratch_dict: Dict[str, Any] = {
"compiler": platforms.DUMMY.id,
"platform": compilers.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "glabel meow\njr $ra",
"diff_label": "meow",
@@ -279,8 +279,8 @@ class ScratchForkTests(BaseTestCase):
slug = scratch.slug
fork_dict = {
"compiler": platforms.DUMMY.id,
"platform": compilers.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"compiler_flags": "-O2",
"source_code": "int func() { return 2; }",
"context": "",
@@ -453,14 +453,14 @@ class ScratchDetailTests(BaseTestCase):
"""
scratch1_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "jr $ra\nnop\n",
}
scratch2_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "jr $ra\nnop\n",
}
@@ -484,14 +484,14 @@ class ScratchDetailTests(BaseTestCase):
"""
scratch1_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": " ",
}
scratch2_dict = {
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": " ",
}
backend/coreapp/tests/test_user.py
+4 -4
View File
@@ -183,8 +183,8 @@ class UserTests(BaseTestCase):
response = self.client.post(
"/api/scratch",
{
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "jr $ra\nnop\n",
},
@@ -219,8 +219,8 @@ class UserTests(BaseTestCase):
response = self.client.post(
"/api/scratch",
{
"compiler": compilers.DUMMY.id,
"platform": platforms.DUMMY.id,
"compiler": compilers.DUMMY_COMPILER.id,
"platform": platforms.DUMMY_PLATFORM.id,
"context": "",
"target_asm": "jr $ra\nnop\n",
},
backend/coreapp/views/platform.py
+4 -4
View File
@@ -8,7 +8,10 @@ from rest_framework.views import APIView
from ..decorators.django import condition
from coreapp.registry import registry
# FIXME: This import is needed to avoid ImportError: cannot import name 'PresetSerializer' from partially initialized module 'coreapp.serializers' (most likely due to a circular import) (/backend/coreapp/serializers.py)
from coreapp import platforms
from coreapp.models.preset import Preset
from coreapp.views.compiler import CompilerDetail
@@ -38,10 +41,7 @@ def single_platform(request: Request, id: str) -> Response:
"""
Gets a platform's basic data
"""
# TODO: if platforms are managed by 'platforms'
# platform = registry.get_platform_by_id(id)
platform = platforms.from_id(id)
platform = registry.get_platform_by_id(id)
if platform:
return Response(
platform.to_json(include_presets=False, include_num_scratches=True)
backend/coreapp/views/register.py
+19 -16
View File
@@ -1,16 +1,10 @@
import logging
import json
from rest_framework import status
from rest_framework.views import APIView
from rest_framework.request import Request
from rest_framework.response import Response
from ..registry import registry
logger = logging.getLogger(__name__)
class Register(APIView):
def post(self, request: Request):
@@ -42,19 +36,28 @@ class Register(APIView):
status=status.HTTP_400_BAD_REQUEST,
)
platforms = payload.get("platforms")
platforms_hash = payload.get("platforms_hash")
compilers = payload.get("compilers")
compilers_hash = payload.get("compilers_hash")
libraries = payload.get("libraries")
libraries_hash = payload.get("libraries_hash")
if compilers is not None and compilers_hash is not None:
registry.add_host(
hostname, port, compilers, compilers_hash, libraries, libraries_hash
)
return Response("Ping/Pong!", status=status.HTTP_200_OK)
if registry.is_known_host(hostname, int(port)):
return Response("Ping/Pong!", status=status.HTTP_200_OK)
return Response("Hello new friend!", status=status.HTTP_201_CREATED)
res = registry.register_host(
hostname,
port,
platforms,
platforms_hash,
compilers,
compilers_hash,
libraries,
libraries_hash,
)
if res is True:
return Response("Registered!", status=status.HTTP_201_CREATED)
elif res is False:
return Response("Failed!", status=status.HTTP_400_BAD_REQUEST)
else:
return Response("OK!", status=status.HTTP_200_OK)
backend/coreapp/views/scratch.py
+2 -3
View File
@@ -8,7 +8,6 @@ from datetime import datetime
from typing import Any, Dict, Optional
import django_filters
from coreapp import compilers, platforms
from django.core.files import File
from django.http import HttpResponse, QueryDict
from rest_framework import filters, mixins, serializers, status
@@ -101,7 +100,7 @@ def diff_compilation(scratch: Scratch, compilation: CompilationResult) -> DiffRe
try:
return DiffWrapper.diff(
scratch.target_assembly,
platforms.from_id(scratch.platform),
registry.get_platform_by_id(scratch.platform),
scratch.diff_label,
bytes(compilation.elf_object),
diff_flags=scratch.diff_flags,
@@ -410,7 +409,7 @@ class ScratchViewSet(
request.data.get("compiler", scratch.compiler)
)
platform = platforms.from_id(scratch.platform)
platform = registry.get_platform_by_id(scratch.platform)
decompilation = DecompilerWrapper.decompile(
"",
docker-compose.yaml
+4
View File
@@ -12,6 +12,10 @@ services:
backend:
build:
context: backend
deploy:
restart_policy:
condition: any
delay: 5s
env_file:
- backend/docker.dev.env
ports:
platforms/TODO.md
+5 -11
View File
@@ -21,17 +21,11 @@ Currently Platforms are defined both in platforms/ and backend/
- move /register 'secret' into environment variables
- delete session if platform becomes unavailable?
- remove DUMMY compilers from 'platforms' ? I think they are only needed for testing backend?
- fix all the broken tests
- move compilation tests into 'platforms' codebase and sub out all compilation/assemble/objdump calls in 'backend'
- (maybe) TCP connection between 'platforms' and 'backend' so we can see when backend is restarted rather than POSTing to /register on a timer?
## Open questions
- Should Platforms belong to both "backend" and "platforms"?
- FOR:
- The frontend needs to know the following thing that the "platforms" code doesnt care about:
- description
- diff_flags
- has_decompiler
- AGAINST:
- Duplicated definitions of Platforms in two places
- Should the 'platform' code only try to hit /register less frequently? We want everything to get up and running when something restarts, so lower time == better
- If we had k8s, we ought to be able to update things without any downtime
- If we had k8s, we ought to be able to update things without any downtime
platforms/main.py
+5 -1
View File
@@ -52,6 +52,9 @@ class LibrariesHandler(tornado.web.RequestHandler):
def register_with_backend():
platforms = [p.to_json() for p in available_platforms()]
platforms_hash = hashlib.sha256(json.dumps(platforms).encode("utf")).hexdigest()
compilers = [c.to_json() for c in available_compilers()]
compilers_hash = hashlib.sha256(json.dumps(compilers).encode("utf")).hexdigest()
@@ -62,6 +65,8 @@ def register_with_backend():
"key": "secret",
"hostname": platform.node(),
"port": settings.PORT,
"platforms": platforms,
"platforms_hash": platforms_hash,
"compilers": compilers,
"compilers_hash": compilers_hash,
"libraries": libraries,
@@ -109,7 +114,6 @@ def main():
concurrent.futures.ThreadPoolExecutor(max_workers=settings.MAX_WORKERS)
)
# start
try:
webapp = tornado.web.Application([*handlers], debug=settings.DEBUG)
server = tornado.httpserver.HTTPServer(webapp)
platforms/src/handlers/compile.py
+9 -7
View File
@@ -41,22 +41,26 @@ class CompileHandler(tornado.web.RequestHandler):
payload = json.loads(self.request.body)
except Exception as e:
logger.error("Exception: %s", e)
return self.write(str(e))
self.set_status(400)
return self.write(json.dumps(dict(error=f"Invalid JSON received: {e}")))
try:
compile_request = CompileRequest.from_dict(payload)
except Exception as e:
logger.error("Exception: %s", e)
return self.write(str(e))
self.set_status(400)
return self.write(
json.dumps(dict(error=f"Error processing CompileRequest: {e}"))
)
logger.debug("compile_request is: %s", compile_request)
compiler = compile_request.compiler
if not compiler.available():
msg = f"Compiler '{compiler.id}' is not available!"
msg = f"Compiler '{compiler.id}' is not currently available!"
logger.warning(msg)
self.set_status(400)
return self.write(msg)
self.set_status(500)
return self.write(json.dumps(dict(error=msg)))
with Sandbox() as sandbox:
context = compile_request.context
@@ -116,8 +120,6 @@ class CompileHandler(tornado.web.RequestHandler):
)
)
logger.info("Hello")
logger.info("libraries_compiler_flags: %s", libraries_compiler_flags)
compile_proc = sandbox.run_subprocess(
cc_cmd,
mounts=([compiler.path] if compiler.platform.id != "dummy" else []),
platforms/src/models/compiler.py
+3 -1
View File
@@ -7,7 +7,7 @@ from pathlib import Path
from .platform import Platform
from .flags import Flags, Language
from ..settings import settings
from ..settings import settings, is_supported_platform
logger = logging.getLogger(__file__)
@@ -36,6 +36,8 @@ class Compiler:
return settings.COMPILER_BASE_PATH / self.platform.id / self.id
def available(self) -> bool:
if not is_supported_platform(self.platform.id):
return False
# consider compiler binaries present if the compiler's directory is found
if not self.path.exists():
logger.warning(f"Compiler {self.id} not found at {self.path}")
platforms/src/models/platform.py
+1
View File
@@ -41,5 +41,6 @@ class Platform:
"name": self.name,
"description": self.description,
"arch": self.arch,
"diff_flags": [flag.to_json() for flag in self.diff_flags],
"has_decompiler": self.has_decompiler,
}