work on forking, caching, etc

CHANGELOG.md

@@ -1,4 +1,5 @@
-# 0.2
+# 8/8/2021
 
 * Improved performance by only sending context to the backend if it was changed from the current version
 
+* Added scratch forking

backend/coreapp/compiler_wrapper.py

@@ -1,4 +1,6 @@
+from typing import Optional
 from coreapp.models import Asm, Assembly, Compilation
+from coreapp import util
 from django.conf import settings
 from django.utils.crypto import get_random_string
 import json
@@ -31,6 +33,16 @@ def load_compilers() -> dict:
 
 compilers = load_compilers()
 
+
+def check_compilation_cache(*args) -> Optional[Compilation]:
+    hash = util.gen_hash(args)
+    return Compilation.objects.filter(hash=hash).first()
+
+def check_assembly_cache(*args) -> Optional[Compilation]:
+    hash = util.gen_hash(args)
+    return Assembly.objects.filter(hash=hash).first()
+
+
 class CompilerWrapper:
     def base_path():
         return settings.COMPILER_BASE_PATH
@@ -45,13 +57,13 @@ class CompilerWrapper:
             .replace("$CC_OPTS", cc_opts) \
             .replace("$AS_OPTS", as_opts)
         compile_command = f"bash -c \"{compile_command}\""
-        # TODO sandbox
 
         logger.debug(f"Compiling: {compile_command}")
 
         return_code = 0
 
         try:
+            # TODO sandbox
             result = subprocess.run(compile_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
             stderr = result.stderr.decode()
 
@@ -106,6 +118,7 @@ class CompilerWrapper:
         logger.debug(f"Objdumping: {objdump_command}")
 
         try:
+            # TODO sandbox
             result = subprocess.run(objdump_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
 
             if result.returncode != 0:
@@ -121,12 +134,19 @@ class CompilerWrapper:
 
     @staticmethod
     def compile_code(compiler: str, cpp_opts: str, as_opts: str, cc_opts: str, code: str, context: str):
-        compiler_path = CompilerWrapper.base_path() / compiler.shortname
+        compiler_path = CompilerWrapper.base_path() / compiler
+
+        compile_cfg = compilers[compiler]
 
         if not compiler_path.exists():
-            logger.error(f"Compiler {compiler.shortname} not found")
+            logger.error(f"Compiler {compiler} not found")
             return (None, "ERROR: Compiler not found")
 
+        cached_compilation = check_compilation_cache(compiler, cpp_opts, as_opts, cc_opts, code, context)
+        if cached_compilation:
+            logger.debug(f"Compilation cache hit!")
+            return (cached_compilation, cached_compilation.stderr)
+
         temp_name = get_random_string(length=8)
         settings.COMPILATION_OBJECTS_PATH.mkdir(exist_ok=True, parents=True)
         code_path = settings.COMPILATION_OBJECTS_PATH / (temp_name + ".c")
@@ -143,7 +163,7 @@ class CompilerWrapper:
 
         # Run compiler
         compile_status, stderr = CompilerWrapper.run_compiler(
-            compiler.compile_cmd,
+            compile_cfg["cc"],
             code_path,
             object_path,
             compiler_path,
@@ -161,19 +181,28 @@ class CompilerWrapper:
             return (None, stderr)
         
         # Store Compilation to db
-        compilation = Compilation(compiler, cpp_opts, as_opts, cc_opts, code, context, object_path)
+        compilation = Compilation(compiler, cpp_opts, as_opts, cc_opts, code, context, object_path, stderr)
         compilation.save()
 
         return (compilation, stderr)
 
     @staticmethod
     def assemble_asm(compiler:str, as_opts: str, asm: Asm, to_overwrite:Assembly = None) -> Assembly:
-        compiler_path = CompilerWrapper.base_path() / compiler.shortname
+        compiler_path = CompilerWrapper.base_path() / compiler
 
         if not compiler_path.exists():
-            logger.error(f"Compiler {compiler.shortname} not found")
+            logger.error(f"Compiler {compiler} not found")
             return "ERROR: Compiler not found"
         
+        # Check the cache if we're not manually re-running an Assembly
+        if not to_overwrite:
+            cached_assembly = check_assembly_cache(compiler, as_opts, asm)
+            if cached_assembly:
+                logger.debug(f"Assembly cache hit!")
+                return cached_assembly
+
+        compiler_cfg = compilers[compiler]
+
         assemblies_path = settings.ASM_OBJECTS_PATH
         assemblies_path.mkdir(exist_ok=True, parents=True)
         
@@ -186,7 +215,7 @@ class CompilerWrapper:
             f.write(ASM_MACROS + asm.data)
 
         assemble_status, stderr = CompilerWrapper.run_assembler(
-            compiler.assemble_cmd,
+            compiler_cfg["as"],
             asm_path,
             object_path,
             compiler_path,
@@ -200,13 +229,12 @@ class CompilerWrapper:
             if object_path.exists():
                 os.remove(object_path)
             return None #f"ERROR: {assemble_status[1]}"
-        
-        # Store Assembly to db
-        assembly = Assembly(compiler, as_opts, source_asm=asm, object=object_path)
 
         if to_overwrite:
             assembly = to_overwrite
             assembly.object = object_path
+        else:
+            assembly = Assembly(compiler, as_opts, source_asm=asm, object=object_path)
 
         assembly.save()
 

backend/coreapp/models.py

@@ -1,3 +1,5 @@
+from django.utils.crypto import get_random_string
+from backend.coreapp.views import gen_scratch_id
 import os
 
 from django.conf import settings
@@ -10,6 +12,14 @@ def asm_objects_path():
 def compilation_objects_path():
     return os.path.join(settings.LOCAL_FILE_DIR, 'compilations')
 
+def gen_scratch_id() -> str:
+    ret = get_random_string(length=5)
+
+    if Scratch.objects.filter(id=ret).exists():
+        return gen_scratch_id()
+
+    return ret
+
 class Profile(models.Model):
     pass
 
@@ -21,6 +31,7 @@ class Asm(models.Model):
         return self.data
 
 class Assembly(models.Model):
+    hash = models.CharField(max_length=64, primary_key=True)
     time = models.DateTimeField(auto_now_add=True)
     compiler = models.CharField(max_length=100)
     as_opts = models.TextField(max_length=1000, blank=True, null=True)
@@ -28,6 +39,7 @@ class Assembly(models.Model):
     object = models.FilePathField(path=settings.ASM_OBJECTS_PATH)
 
 class Compilation(models.Model):
+    hash = models.CharField(max_length=64, primary_key=True)
     time = models.DateTimeField(auto_now_add=True)
     compiler = models.CharField(max_length=100)
     cpp_opts = models.TextField(max_length=1000, blank=True, null=True)
@@ -36,9 +48,10 @@ class Compilation(models.Model):
     source_code = models.TextField()
     context = models.TextField(blank=True)
     object = models.FilePathField(path=settings.COMPILATION_OBJECTS_PATH)
+    stderr = models.TextField(blank=True, null=True)
 
 class Scratch(models.Model):
-    slug = models.SlugField(primary_key=True)
+    slug = models.SlugField(primary_key=True, default=gen_scratch_id)
     creation_time = models.DateTimeField(auto_now_add=True)
     last_updated = models.DateTimeField(auto_now=True)
     compiler = models.CharField(max_length=100)

backend/coreapp/urls.py

@@ -3,8 +3,8 @@ from django.urls import path
 from . import views
 
 urlpatterns = [
-    path('', views.index, name='index'),
     path('scratch', views.scratch, name='scratch'),
     path('scratch/<slug:slug>', views.scratch, name='scratch'),
     path('scratch/<slug:slug>/compile', views.compile, name='compile_scratch'),
+    path('scratch/<slug:slug>/fork', views.fork, name='fork_scratch'),
 ]

backend/coreapp/util.py

@@ -0,0 +1,4 @@
+import hashlib
+
+def gen_hash(dict):
+    return hashlib.sha256(str(dict).encode('utf-8')).hexdigest()

backend/coreapp/views.py

@@ -2,7 +2,6 @@ from coreapp.asm_diff_wrapper import AsmDifferWrapper
 from coreapp.m2c_wrapper import M2CWrapper
 from coreapp.compiler_wrapper import CompilerWrapper
 from coreapp.serializers import ScratchSerializer
-from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from rest_framework import status
 from rest_framework.response import Response
@@ -13,9 +12,7 @@ import logging
 import hashlib
 
 from .models import Profile, Asm, Scratch
-
-def index(request):
-    return HttpResponse("This is the index page.")
+from coreapp.models import gen_scratch_id
 
 def get_db_asm(request_asm) -> Asm:
     h = hashlib.sha256(request_asm.encode()).hexdigest()
@@ -30,7 +27,6 @@ def get_db_asm(request_asm) -> Asm:
     
     return ret
 
-
 @api_view(["GET", "POST", "PATCH"])
 def scratch(request, slug=None):
     """
@@ -71,7 +67,8 @@ def scratch(request, slug=None):
         if "target_asm" not in data:
             return Response({"error": "Missing target_asm"}, status=status.HTTP_400_BAD_REQUEST)
 
-        data["slug"] = get_random_string(length=5)
+        # TODO remove this - should be done automatically but not sure if the serializer requires it
+        data["slug"] = gen_scratch_id()
 
         asm = get_db_asm(data["target_asm"])
         del data["target_asm"]
@@ -90,8 +87,6 @@ def scratch(request, slug=None):
 
         serializer = ScratchSerializer(data=data)
         if serializer.is_valid():
-            if serializer.context:
-                serializer.original_context = serializer.context
             serializer.save()
             return Response(serializer.data, status=status.HTTP_201_CREATED)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
@@ -156,3 +151,38 @@ def compile(request, slug):
     }
         
     return Response(response_obj)
+
+@api_view(["POST"])
+def fork(request, parent_slug):
+    required_params = ["compiler", "cpp_opts", "as_opts", "cc_opts", "source_code", "context"]
+
+    for param in required_params:
+        if param not in request.data:
+            return Response({"error": f"Missing parameter: {param}"}, status=status.HTTP_400_BAD_REQUEST)
+
+    parent_scratch = Scratch.objects.filter(slug=parent_slug).first()
+
+    if not parent_scratch:
+        return Response({"error": "Parent scratch does not exist"}, status=status.HTTP_400_BAD_REQUEST)
+
+    # TODO validate
+    compiler = request.data["compiler"]
+    cpp_opts = request.data["cpp_opts"]
+    as_opts = request.data["as_opts"]
+    cc_opts = request.data["cc_opts"]
+    code = request.data["source_code"]
+    context = request.data["context"]
+
+    new_scratch = Scratch(
+        compiler=compiler,
+        cpp_opts=cpp_opts,
+        as_opts=as_opts,
+        cc_opts=cc_opts,
+        target_assembly=parent_scratch.target_assembly,
+        source_code=code,
+        context=context,
+        original_context=parent_scratch.original_context,
+        parent=parent_scratch,
+    )
+    new_scratch.save()
+    return Response(ScratchSerializer(new_scratch).data, status=status.HTTP_201_CREATED)