Overhaul download py (#855)

* Add translations for ps1/maspsx compiler set

* Tell clang to compile and not link to remove need for lld binary

* Assume msdos assembler lives on PATH

* Overhaul download.py

* Attempt to fix docker ci

* Appease black

* Whoops, don't forget to detach

* Fix path to download.py

* fix perms (hack?)

* Add --platforms and --compilers args to download.py

* Try out --compiler dir arg for docker in ci

* Try to ignore that 500 error about image manifests

* it's --compilerS-dir

* Try again

* Try building our own podman (fallback to vfs when rootless)

* install distro 'podman' to get configuration (fixes fuse)

* mount /dev/fuse in ci

* dumb bugfix, .download_cache

* split compilers config out of download.py

* update poetry.lock

* Add psyq3.3 + psyq4.4 and migrate psyq4.3 scratches to psyq4.4

* docker_entrypoint.sh cleanup

* Try to protect against 'locating item named "manifest" for image with ID'

* better podman sanity check

* appease black

* use 'requests.get()' to grab remote image digest

* Update compilers repo to 'decompme/compilers'

* Fix description for psyq3.6

.github/workflows/ci.yml

@@ -52,6 +52,10 @@ jobs:
             libnl-route-3-dev \
             libncurses5 \
             protobuf-compiler
+      - name: Install msdos assembler
+        run: |-
+          wget https://github.com/OmniBlade/binutils-gdb/releases/download/omf-build/omftools.tar.gz
+          sudo tar xvzf omftools.tar.gz -C /usr/bin jwasm omf-nm omf-objdump
       - name: Install apt dependencies (cached)
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
@@ -164,14 +168,19 @@ jobs:
             -t decompme_backend \
             --build-arg ENABLE_GC_WII_SUPPORT=YES \
             --build-arg ENABLE_PS1_SUPPORT=YES \
-            --build-arg ENABLE_SATURN_SUPPORT=YES
+            --build-arg ENABLE_SATURN_SUPPORT=YES \
+            --build-arg ENABLE_MSDOS_SUPPORT=YES
       - name: Run tests
         run: |-
           mkdir -p sandbox && chmod 777 sandbox
           mkdir -p local_files && chmod 777 local_files
-          docker run \
+          mkdir -p compilers && chmod 777 compilers
+          container_id=$(docker run \
+            --detach \
             -v $(pwd):/decomp.me \
             -v $(pwd)/local_files:/local_files \
+            -v $(pwd)/compilers:/compilers \
+            --device /dev/fuse \
             --security-opt apparmor=unconfined \
             --security-opt seccomp=unconfined \
             --entrypoint /bin/bash \
@@ -181,8 +190,11 @@ jobs:
             -e USE_SANDBOX_JAIL=on \
             -e SANDBOX_DISABLE_PROC=true \
             -e TIMEOUT_SCALE_FACTOR=10 \
-            decompme_backend -c 'cd /decomp.me/backend && \
+            decompme_backend -c 'podman system service --time=0 unix:///tmp/podman.sock')
+
+          docker exec ${container_id} /bin/bash -c 'cd /decomp.me/backend && \
             poetry install && \
+            poetry run compilers/download.py --compilers-dir ${COMPILER_BASE_PATH} --podman && \
             poetry run python manage.py test'
 
   frontend_lint: